From d93a6bfb523a038dc66de4ae98a81df2b4e781cb Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 29 May 2026 08:37:09 +0000 Subject: [PATCH] darwin: add E2E CI test against Headscale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds nix/darwin/tests/ci/, a self-contained test that: - boots Headscale on 127.0.0.1:8080 (HTTP, sqlite, ephemeral state, embedded DERP server) - creates two preauth keys (alpha and beta users) - applies a darwinConfiguration via `sudo nix run github:LnL7/nix-darwin -- switch` against services.tailscales.{alpha,beta} - waits for the per-instance daemon agents to load and their sockets to answer - runs `tailscale- up --reset --auth-key` and polls each instance for BackendState=Running - asserts per-instance UserID, socket, and state-file isolation - tears down the agents and Headscale on exit The test flake lives separately so the main flake stays free of a nix-darwin input — users importing darwinModules.tailscales are not forced to pull nix-darwin transitively. `nix.enable = false` lets the config coexist with the DeterminateSystems Nix install on the runner. Wires the test into a new .github/workflows/nix.yml: a cheap flake-check-linux job gates `nix flake check --no-build` (catches the existing darwin-eval and NixOS module regressions), and darwin-e2e runs the orchestration on macos-latest only after the eval gate passes. Failed runs upload Tailscale and Headscale log tails as artifacts. Updates nix/darwin/tests/README.md to document the new harness and how to run it locally on a Mac. Signed-off-by: Kristoffer Dalby --- .github/workflows/nix.yml | 57 ++++++++ nix/darwin/tests/README.md | 21 ++- nix/darwin/tests/ci/flake.lock | 96 ++++++++++++++ nix/darwin/tests/ci/flake.nix | 54 ++++++++ nix/darwin/tests/ci/headscale.yaml | 59 +++++++++ nix/darwin/tests/ci/run.sh | 204 +++++++++++++++++++++++++++++ 6 files changed, 489 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/nix.yml create mode 100644 nix/darwin/tests/ci/flake.lock create mode 100644 nix/darwin/tests/ci/flake.nix create mode 100644 nix/darwin/tests/ci/headscale.yaml create mode 100755 nix/darwin/tests/ci/run.sh diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 000000000..4f233824a --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,57 @@ +name: nix + +on: + pull_request: + paths: + - "flake.nix" + - "flakehashes.json" + - "nix/**" + - ".github/workflows/nix.yml" + push: + branches: + - main + paths: + - "flake.nix" + - "flakehashes.json" + - "nix/**" + - ".github/workflows/nix.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + flake-check-linux: + name: flake check (linux) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: DeterminateSystems/nix-installer-action@c5a866b6ab867e88becbed4467b93592bce69f8a # v21 + # Eval-only so the heavyweight NixOS VM tests do not block the gate. + # The VM tests still run on demand via `nix flake check` locally. + - name: nix flake check (no-build) + run: nix flake check --no-build + + darwin-e2e: + name: darwin E2E (Headscale + nix-darwin) + runs-on: macos-latest + needs: flake-check-linux + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: DeterminateSystems/nix-installer-action@c5a866b6ab867e88becbed4467b93592bce69f8a # v21 + - name: run E2E test + working-directory: nix/darwin/tests/ci + run: bash run.sh + - name: upload logs on failure + if: failure() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: darwin-e2e-logs + path: | + /tmp/ts-ci/headscale.log + /Users/runner/Library/Logs/Tailscale-*.log diff --git a/nix/darwin/tests/README.md b/nix/darwin/tests/README.md index 1f21078ea..0419b0aed 100644 --- a/nix/darwin/tests/README.md +++ b/nix/darwin/tests/README.md @@ -15,8 +15,25 @@ Runs on Linux and macOS via: nix flake check ``` -There is no real darwin VM available in `nix-build`, so this is as close as -we can get without spinning up a Mac. +## Automated: end-to-end (`ci/`) + +`ci/` contains a self-contained test that brings up a real Headscale on +loopback, applies a sample nix-darwin configuration via +`nix run github:LnL7/nix-darwin`, and verifies that two userspace +tailscaled instances register against separate Headscale users and stay +isolated. + +It runs on every PR via `.github/workflows/nix.yml` on a `macos-latest` +GitHub Actions runner. To run it locally on a Mac with Nix installed: + +``` +cd nix/darwin/tests/ci +bash run.sh +``` + +`run.sh` cleans up after itself via a `trap` (boots out the LaunchAgents +and kills Headscale). The first run is slow because it builds Tailscale +and Headscale from source. ## Manual: integration on a Mac diff --git a/nix/darwin/tests/ci/flake.lock b/nix/darwin/tests/ci/flake.lock new file mode 100644 index 000000000..29f384a19 --- /dev/null +++ b/nix/darwin/tests/ci/flake.lock @@ -0,0 +1,96 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "nix-darwin": { + "inputs": { + "nixpkgs": [ + "parent", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1779036909, + "narHash": "sha256-zXcwYQGCT6pzinK+1dBB2ekTVtfxGZAapb3Evdcu4fY=", + "owner": "LnL7", + "repo": "nix-darwin", + "rev": "56c666e108467d87d13508936aade6d567f2a501", + "type": "github" + }, + "original": { + "owner": "LnL7", + "repo": "nix-darwin", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1772736753, + "narHash": "sha256-au/m3+EuBLoSzWUCb64a/MZq6QUtOV8oC0D9tY2scPQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "917fec990948658ef1ccd07cef2a1ef060786846", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "parent": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs", + "systems": "systems" + }, + "locked": { + "path": "../../../..", + "type": "path" + }, + "original": { + "path": "../../../..", + "type": "path" + }, + "parent": [] + }, + "root": { + "inputs": { + "nix-darwin": "nix-darwin", + "parent": "parent" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/darwin/tests/ci/flake.nix b/nix/darwin/tests/ci/flake.nix new file mode 100644 index 000000000..992080b9f --- /dev/null +++ b/nix/darwin/tests/ci/flake.nix @@ -0,0 +1,54 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause +# +# Test flake for the macOS multi-instance Tailscale module. +# Consumed by run.sh on a macOS CI runner. Lives in its own flake so the +# parent tailscale flake does not take a runtime dependency on nix-darwin. +{ + inputs = { + parent.url = "path:../../../.."; + nix-darwin = { + url = "github:LnL7/nix-darwin"; + inputs.nixpkgs.follows = "parent/nixpkgs"; + }; + }; + + outputs = { + nix-darwin, + parent, + ... + }: let + mkSystem = system: + nix-darwin.lib.darwinSystem { + inherit system; + modules = [ + parent.darwinModules.default + ({...}: { + system.primaryUser = "runner"; + # Required by recent nix-darwin to bound state-version compat. + system.stateVersion = 5; + # DeterminateSystems Nix manages the install itself; let + # nix-darwin stand aside on /etc/nix/nix.conf and friends. + nix.enable = false; + services.tailscales = { + alpha = { + enable = true; + authKeyFile = "/tmp/ts-ci/alpha.key"; + extraUpFlags = ["--login-server=http://127.0.0.1:8080"]; + }; + beta = { + enable = true; + authKeyFile = "/tmp/ts-ci/beta.key"; + extraUpFlags = ["--login-server=http://127.0.0.1:8080"]; + }; + }; + }) + ]; + }; + in { + darwinConfigurations = { + ci-mac-aarch64 = mkSystem "aarch64-darwin"; + ci-mac-x86_64 = mkSystem "x86_64-darwin"; + }; + }; +} diff --git a/nix/darwin/tests/ci/headscale.yaml b/nix/darwin/tests/ci/headscale.yaml new file mode 100644 index 000000000..b564ef2e3 --- /dev/null +++ b/nix/darwin/tests/ci/headscale.yaml @@ -0,0 +1,59 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause +# +# Minimal Headscale config for the macOS E2E test. +# HTTP only (loopback), embedded SQLite, ephemeral state under /tmp/ts-ci. +server_url: http://127.0.0.1:8080 +listen_addr: 127.0.0.1:8080 +metrics_listen_addr: 127.0.0.1:9090 +grpc_listen_addr: 127.0.0.1:50443 +grpc_allow_insecure: true + +unix_socket: /tmp/ts-ci/headscale.sock +unix_socket_permission: "0770" + +private_key_path: /tmp/ts-ci/private.key +noise: + private_key_path: /tmp/ts-ci/noise_private.key + +database: + type: sqlite3 + sqlite: + path: /tmp/ts-ci/headscale.sqlite + +prefixes: + v4: 100.64.0.0/10 + v6: fd7a:115c:a1e0::/48 + allocation: sequential + +derp: + server: + enabled: true + region_id: 999 + region_code: "ts-ci" + region_name: "Headscale embedded DERP" + stun_listen_addr: "127.0.0.1:3478" + private_key_path: /tmp/ts-ci/derp_server_private.key + automatically_add_embedded_derp_region: true + ipv4: 127.0.0.1 + ipv6: "" + urls: [] + paths: [] + auto_update_enabled: false + +disable_check_updates: true +ephemeral_node_inactivity_timeout: 30m + +log: + format: text + level: info + +dns: + override_local_dns: false + magic_dns: false + base_domain: ts-ci.example + nameservers: + global: [] + +policy: + mode: database diff --git a/nix/darwin/tests/ci/run.sh b/nix/darwin/tests/ci/run.sh new file mode 100755 index 000000000..d4011d2b6 --- /dev/null +++ b/nix/darwin/tests/ci/run.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause +# +# End-to-end test for the macOS Tailscale module. +# Runs Headscale on loopback, applies a nix-darwin config that defines +# two userspace tailscaled instances, and verifies each one authenticates +# against its own Headscale user without leaking state to the other. +# +# Designed for a clean macos-latest GitHub Actions runner. Safe to run +# locally on a Mac that already has Nix installed. + +set -euo pipefail + +readonly STATE_DIR="/tmp/ts-ci" +readonly HEADSCALE_LOG="${STATE_DIR}/headscale.log" +readonly TAILSCALE_LOG_DIR="${HOME}/Library/Logs" +readonly INSTANCES=(alpha beta) + +# Pick the darwinConfiguration matching this host. +arch=$(uname -m) +case "$arch" in + arm64) readonly DARWIN_CFG="ci-mac-aarch64" ;; + x86_64) readonly DARWIN_CFG="ci-mac-x86_64" ;; + *) echo "unsupported arch $arch" >&2; exit 1 ;; +esac + +log() { printf '\n=== %s ===\n' "$*"; } +fail() { printf '\nFAIL: %s\n' "$*" >&2; dump_logs; exit 1; } + +dump_logs() { + # `>&2 2>/dev/null` redirects stdout to the original stderr, then + # silences stderr — the opposite order silently sends everything to + # /dev/null. + printf '\n--- headscale.log (tail) ---\n' >&2 + tail -n 100 "$HEADSCALE_LOG" >&2 2>/dev/null || true + for inst in "${INSTANCES[@]}"; do + printf '\n--- Tailscale-%s.log (tail) ---\n' "$inst" >&2 + tail -n 100 "${TAILSCALE_LOG_DIR}/Tailscale-${inst}.log" >&2 2>/dev/null || true + done +} + +cleanup() { + local rc=$? + log "cleanup" + for inst in "${INSTANCES[@]}"; do + launchctl bootout "gui/$(id -u)" \ + "${HOME}/Library/LaunchAgents/com.tailscale.tailscaled-${inst}.plist" \ + 2>/dev/null || true + launchctl bootout "gui/$(id -u)" \ + "${HOME}/Library/LaunchAgents/com.tailscale.tailscale-${inst}-bootstrap.plist" \ + 2>/dev/null || true + done + if [[ -n "${HEADSCALE_PID:-}" ]]; then + kill "$HEADSCALE_PID" 2>/dev/null || true + wait "$HEADSCALE_PID" 2>/dev/null || true + fi + exit "$rc" +} +trap cleanup EXIT INT TERM + +main() { + log "workspace" + mkdir -p "$STATE_DIR" "$TAILSCALE_LOG_DIR" + + # Build headscale and jq up front. Each subsequent invocation goes + # directly to the resolved store-path binary — far cheaper than + # spawning `nix shell` per call. Use the ^bin output selector since + # jq is multi-output (bin + man + …) and `--print-out-paths` lists + # every output otherwise. + log "fetch headscale + jq" + HEADSCALE=$(nix build --quiet --no-link --print-out-paths \ + 'nixpkgs#headscale')/bin/headscale + JQ=$(nix build --quiet --no-link --print-out-paths \ + 'nixpkgs#jq^bin')/bin/jq + + log "start headscale" + "$HEADSCALE" serve -c "${PWD}/headscale.yaml" \ + > "$HEADSCALE_LOG" 2>&1 & + HEADSCALE_PID=$! + + log "wait for headscale" + for _ in $(seq 1 60); do + if "$HEADSCALE" -c "${PWD}/headscale.yaml" users list >/dev/null 2>&1; then + break + fi + sleep 1 + done + "$HEADSCALE" -c "${PWD}/headscale.yaml" users list \ + >/dev/null 2>&1 || fail "headscale did not become ready" + + log "create users + preauth keys" + for inst in "${INSTANCES[@]}"; do + # Headscale 0.28's `preauthkeys create --user` expects a numeric ID, + # not a name. Create the user, then look its ID up by name. + "$HEADSCALE" -c "${PWD}/headscale.yaml" users create "$inst" >/dev/null + user_id=$("$HEADSCALE" -c "${PWD}/headscale.yaml" users list --output json \ + | "$JQ" -r ".[] | select(.name==\"$inst\") | .id") + [[ -n "$user_id" ]] || fail "could not resolve user id for $inst" + key=$("$HEADSCALE" -c "${PWD}/headscale.yaml" \ + preauthkeys create --reusable --expiration 1h --user "$user_id" \ + --output json | "$JQ" -r .key) + [[ -n "$key" ]] || fail "empty preauth key for $inst" + printf '%s' "$key" > "${STATE_DIR}/${inst}.key" + chmod 600 "${STATE_DIR}/${inst}.key" + done + + log "apply nix-darwin config (${DARWIN_CFG})" + # Recent nix-darwin requires system activation to run as root. Invoke + # the user's `nix` (DeterminateSystems install path may not be on + # root's default PATH) and preserve the env so flake-fetching and + # cache lookups use the same daemon. + nix_bin=$(command -v nix) + sudo --preserve-env=HOME,USER \ + "$nix_bin" run --quiet github:LnL7/nix-darwin -- switch \ + --flake "${PWD}#${DARWIN_CFG}" + + # darwin-rebuild updates the system profile, but the running shell's + # PATH was captured at startup. Pick up the freshly-installed CLI + # wrappers (tailscale-alpha, tailscale-beta) before exercising them. + user=$(id -un) + export PATH="/run/current-system/sw/bin:/etc/profiles/per-user/${user}/bin:${PATH}" + + log "wait for LaunchAgents" + for inst in "${INSTANCES[@]}"; do + for _ in $(seq 1 60); do + if launchctl print "gui/$(id -u)/com.tailscale.tailscaled-${inst}" \ + >/dev/null 2>&1; then + break + fi + sleep 1 + done + launchctl print "gui/$(id -u)/com.tailscale.tailscaled-${inst}" \ + >/dev/null 2>&1 || fail "agent tailscaled-${inst} never loaded" + done + + log "wait for daemon socket" + for inst in "${INSTANCES[@]}"; do + sock="${HOME}/Library/Caches/Tailscale-${inst}/tsd.sock" + for _ in $(seq 1 60); do + # `status` (no --json) exits non-zero before login, so we'd loop + # forever. `--json` only checks that the backend is reachable. + [[ -S "$sock" ]] && "tailscale-${inst}" status --json >/dev/null 2>&1 && break + sleep 1 + done + "tailscale-${inst}" status --json >/dev/null 2>&1 \ + || fail "${inst} daemon socket never became responsive" + done + + # The module's bootstrap LaunchAgent should have run `tailscale up` + # automatically on activation. In CI the launchctl gui domain doesn't + # always cooperate when activation runs under sudo, so re-invoke `up` + # explicitly — idempotent against an instance the bootstrap already + # authenticated, and the canonical fallback for users debugging by + # hand. The verify-Running step below catches both paths. + log "authenticate instances" + for inst in "${INSTANCES[@]}"; do + key=$(cat "${STATE_DIR}/${inst}.key") + "tailscale-${inst}" up --reset \ + --auth-key="$key" \ + --login-server=http://127.0.0.1:8080 \ + --hostname="ci-${inst}" + done + + log "wait for BackendState=Running" + for inst in "${INSTANCES[@]}"; do + for _ in $(seq 1 120); do + state=$("tailscale-${inst}" status --json 2>/dev/null \ + | "$JQ" -r '.BackendState' 2>/dev/null || true) + [[ "$state" == "Running" ]] && break + sleep 1 + done + [[ "$state" == "Running" ]] || fail "${inst} stuck in state=${state:-}" + done + + log "verify per-instance identity" + alpha_user=$("tailscale-alpha" status --json | "$JQ" -r '.Self.UserID') + beta_user=$("tailscale-beta" status --json | "$JQ" -r '.Self.UserID') + [[ -n "$alpha_user" && -n "$beta_user" ]] \ + || fail "missing Self.UserID on one of the instances" + [[ "$alpha_user" != "$beta_user" ]] \ + || fail "alpha and beta share the same UserID (${alpha_user}) — isolation broken" + + log "verify socket isolation" + for inst in "${INSTANCES[@]}"; do + sock="${HOME}/Library/Caches/Tailscale-${inst}/tsd.sock" + [[ -S "$sock" ]] || fail "socket missing: $sock" + done + + log "verify state isolation" + for inst in "${INSTANCES[@]}"; do + state_file="${HOME}/Library/Application Support/Tailscale-${inst}/tailscaled.state" + [[ -f "$state_file" ]] || fail "state file missing: $state_file" + done + + alpha_state=$(shasum -a 256 "${HOME}/Library/Application Support/Tailscale-alpha/tailscaled.state" | awk '{print $1}') + beta_state=$(shasum -a 256 "${HOME}/Library/Application Support/Tailscale-beta/tailscaled.state" | awk '{print $1}') + [[ "$alpha_state" != "$beta_state" ]] \ + || fail "alpha and beta tailscaled.state are byte-identical — isolation broken" + + log "PASS" +} + +main "$@"