.github, tstest/natlab/vmtest: replace old VM runner job with natlab tests

The "vm" CI job ran a single test (TestRunUbuntu2404 from
tstest/integration/vms) on a privileged self-hosted runner. Its coverage
is nearly all redundant with the modern natlab vmtest suite, which boots
real Ubuntu VMs and already exercises connectivity, kernel TUN, SSH,
Taildrop, ACME, and OS DNS integration on GitHub-hosted runners.

The two things it tested that natlab didn't are added back as natlab
tests so the runner can be decommissioned:

TestUbuntuSystemdUnit runs tailscaled via the stock systemd unit that
Linux packages ship (cmd/tailscaled/tailscaled.service with
tailscaled.defaults as its EnvironmentFile) instead of launching the
binary directly, verifying the unit's directives and its Type=notify
readiness handshake.

TestDNSExtraRecordsSearchDomains verifies that control-plane DNS
ExtraRecords and search domains are resolvable through the guest's OS
resolver (libc to systemd-resolved to quad-100).

Updates #13038

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
Change-Id: I8d0dfb8b8153289e7ca78f3af03dfece9497bfe8

.github/workflows/test.yml

@@ -365,30 +365,6 @@ jobs:
       working-directory: src
       run: ./tool/go test $(./tool/go run ./tool/listpkgs --has-root-tests)
 
-  vm:
-    needs: gomod-cache
-    runs-on: ["self-hosted", "linux", "vm"]
-    # VM tests run with some privileges, don't let them run on 3p PRs.
-    if: github.repository == 'tailscale/tailscale'
-    steps:
-    - name: checkout
-      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      with:
-        path: src
-    - name: Restore Go module cache
-      uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
-      with:
-        path: gomodcache
-        key: ${{ needs.gomod-cache.outputs.cache-key }}
-        enableCrossOsArchive: true
-    - name: Run VM tests
-      working-directory: src
-      run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2404
-      env:
-        HOME: "/var/lib/ghrunner/home"
-        TMPDIR: "/tmp"
-        XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
-
   cross: # cross-compile checks, build only.
     needs: gomod-cache
     strategy:
@@ -912,7 +888,6 @@ jobs:
       - test
       - windows
       - macos
-      - vm
       - cross
       - ios
       - wasm
@@ -958,7 +933,6 @@ jobs:
       - test
       - windows
       - macos
-      - vm
       - cross
       - ios
       - wasm
@@ -1008,7 +982,6 @@ jobs:
       - test
       - windows
       - macos
-      - vm
       - wasm
       - fuzz
       - race-root-integration

tstest/natlab/vmtest/cloudinit.go

@@ -6,6 +6,7 @@
 import (
 	"crypto/ed25519"
 	"crypto/rand"
+	"encoding/base64"
 	"encoding/pem"
 	"fmt"
 	"os"
@@ -110,6 +111,10 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 		ud.WriteString(fmt.Sprintf("    ssh_authorized_keys:\n      - %s\n", strings.TrimSpace(string(pubkey))))
 	}
 
+	if n.systemdUnit {
+		e.writeSystemdUnitFiles(&ud, n)
+	}
+
 	ud.WriteString("runcmd:\n")
 
 	// Remove the default route from the debug NIC (enp0s4) so traffic goes through vnet.
@@ -130,12 +135,21 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 		ud.WriteString("  - [\"sysctl\", \"-w\", \"net.ipv6.conf.all.forwarding=1\"]\n")
 	}
 
-	// Start tailscaled in the background. --statedir provides a VarRoot so
-	// features like Taildrop (which needs a place to stash incoming files)
-	// have a directory to work with.
-	ud.WriteString("  - [\"mkdir\", \"-p\", \"/var/lib/tailscale\"]\n")
-	fmt.Fprintf(&ud, "  - [\"/bin/sh\", \"-c\", \"%s/usr/local/bin/tailscaled --state=mem: --statedir=/var/lib/tailscale &\"]\n", tailscaledEnvPrefix(n))
-	ud.WriteString("  - [\"sleep\", \"2\"]\n")
+	// Start tailscaled, either via the stock systemd unit or directly in
+	// the background. --statedir provides a VarRoot so features like
+	// Taildrop (which needs a place to stash incoming files) have a
+	// directory to work with.
+	if n.systemdUnit {
+		// The unit's ExecStart runs /usr/sbin/tailscaled.
+		ud.WriteString("  - [\"cp\", \"/usr/local/bin/tailscaled\", \"/usr/sbin/tailscaled\"]\n")
+		ud.WriteString("  - [\"systemctl\", \"daemon-reload\"]\n")
+		// Type=notify makes this block until tailscaled reports readiness.
+		ud.WriteString("  - [\"systemctl\", \"start\", \"tailscaled.service\"]\n")
+	} else {
+		ud.WriteString("  - [\"mkdir\", \"-p\", \"/var/lib/tailscale\"]\n")
+		fmt.Fprintf(&ud, "  - [\"/bin/sh\", \"-c\", \"%s/usr/local/bin/tailscaled --state=mem: --statedir=/var/lib/tailscale &\"]\n", tailscaledEnvPrefix(n))
+		ud.WriteString("  - [\"sleep\", \"2\"]\n")
+	}
 
 	// Start tta (Tailscale Test Agent).
 	ud.WriteString("  - [\"/bin/sh\", \"-c\", \"/usr/local/bin/tta &\"]\n")
@@ -225,6 +239,40 @@ func (e *Env) generateFreeBSDUserData(n *Node) string {
 	return ud.String()
 }
 
+// writeSystemdUnitFiles appends a cloud-init write_files section that
+// installs the stock tailscaled systemd unit from the source tree, along
+// with the packaging's /etc/default/tailscaled EnvironmentFile (plus any
+// per-node TailscaledEnv variables). File contents are base64-encoded to
+// sidestep YAML quoting.
+func (e *Env) writeSystemdUnitFiles(ud *strings.Builder, n *Node) {
+	modRoot, err := findModRoot()
+	if err != nil {
+		e.t.Fatalf("finding module root for tailscaled.service: %v", err)
+	}
+	unit, err := os.ReadFile(filepath.Join(modRoot, "cmd/tailscaled/tailscaled.service"))
+	if err != nil {
+		e.t.Fatalf("reading tailscaled.service: %v", err)
+	}
+	defaults, err := os.ReadFile(filepath.Join(modRoot, "cmd/tailscaled/tailscaled.defaults"))
+	if err != nil {
+		e.t.Fatalf("reading tailscaled.defaults: %v", err)
+	}
+	var envFile strings.Builder
+	envFile.Write(defaults)
+	for _, env := range n.vnetNode.Env() {
+		fmt.Fprintf(&envFile, "%s=%q\n", env.Key, env.Value)
+	}
+
+	ud.WriteString("write_files:\n")
+	writeFile := func(path string, content []byte) {
+		fmt.Fprintf(ud, "  - path: %s\n", path)
+		fmt.Fprintf(ud, "    encoding: b64\n")
+		fmt.Fprintf(ud, "    content: %s\n", base64.StdEncoding.EncodeToString(content))
+	}
+	writeFile("/etc/systemd/system/tailscaled.service", unit)
+	writeFile("/etc/default/tailscaled", []byte(envFile.String()))
+}
+
 func tailscaledEnvPrefix(n *Node) string {
 	env := n.vnetNode.Env()
 	if len(env) == 0 {

tstest/natlab/vmtest/dns_test.go

@@ -0,0 +1,56 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstest"
+	"tailscale.com/tstest/natlab/vmtest"
+	"tailscale.com/tstest/natlab/vnet"
+	"tailscale.com/types/dnstype"
+)
+
+// TestDNSExtraRecordsSearchDomains verifies that control-plane DNS config
+// makes it all the way into an Ubuntu guest's OS resolver: an ExtraRecords
+// entry ("extratest.record" → 1.2.3.4) resolves via libc (getent), and the
+// "record" search domain lets the bare name "extratest" resolve too. That
+// requires tailscaled to have plumbed MagicDNS routes and search domains
+// into systemd-resolved.
+func TestDNSExtraRecordsSearchDomains(t *testing.T) {
+	env := vmtest.New(t, vmtest.ControlDNS("tailnet.test", &tailcfg.DNSConfig{
+		Proxied: true,
+		Domains: []string{"record"},
+		Routes:  map[string][]*dnstype.Resolver{"record": nil},
+		ExtraRecords: []tailcfg.DNSRecord{
+			{Name: "extratest.record", Type: "A", Value: "1.2.3.4"},
+		},
+	}))
+	node := env.AddNode("node",
+		env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT),
+		vmtest.OS(vmtest.Ubuntu2404))
+	env.Start()
+
+	for _, name := range []string{"extratest.record", "extratest"} {
+		// Retry for a bit: tailscaled applies the DNS config to
+		// systemd-resolved asynchronously after coming up.
+		if err := tstest.WaitFor(30*time.Second, func() error {
+			out, err := env.SSHExec(node, "getent hosts "+name)
+			if err != nil {
+				return fmt.Errorf("getent hosts %s: %v (%s)", name, err, strings.TrimSpace(out))
+			}
+			if !strings.Contains(out, "1.2.3.4") {
+				return fmt.Errorf("getent hosts %s = %q, want it to contain 1.2.3.4", name, strings.TrimSpace(out))
+			}
+			return nil
+		}); err != nil {
+			out, _ := env.SSHExec(node, "resolvectl status; cat /etc/resolv.conf")
+			t.Fatalf("%v\nresolver state:\n%s", err, out)
+		}
+	}
+}

tstest/natlab/vmtest/systemd_test.go

@@ -0,0 +1,48 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"strings"
+	"testing"
+
+	"tailscale.com/tstest/natlab/vmtest"
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+// TestUbuntuSystemdUnit runs tailscaled on an Ubuntu node via the stock
+// systemd unit that Linux packages ship (cmd/tailscaled/tailscaled.service)
+// rather than launching the binary directly, exercising the unit's
+// directives (EnvironmentFile, RuntimeDirectory, StateDirectory) and its
+// Type=notify readiness handshake. Env.Start already asserts that the node
+// reaches the Running backend state under that unit.
+func TestUbuntuSystemdUnit(t *testing.T) {
+	env := vmtest.New(t)
+	node := env.AddNode("node",
+		env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT),
+		vmtest.OS(vmtest.Ubuntu2404),
+		vmtest.SystemdUnit())
+	env.Start()
+
+	// With Type=notify, systemd only reports the unit active after
+	// tailscaled sends READY=1, so this also verifies the sd_notify
+	// handshake worked.
+	out, err := env.SSHExec(node, "systemctl is-active tailscaled.service")
+	if err != nil {
+		t.Fatalf("systemctl is-active: %v\n%s", err, out)
+	}
+	if got := strings.TrimSpace(out); got != "active" {
+		t.Fatalf("tailscaled.service state = %q, want %q", got, "active")
+	}
+
+	// Verify the tailscaled that TTA talked to is the one systemd runs,
+	// not a stray process started some other way.
+	out, err = env.SSHExec(node, `test "$(systemctl show -p MainPID --value tailscaled.service)" = "$(pidof tailscaled)" && echo match`)
+	if err != nil {
+		t.Fatalf("MainPID check: %v\n%s", err, out)
+	}
+	if got := strings.TrimSpace(out); got != "match" {
+		t.Fatalf("MainPID check output = %q, want %q", got, "match")
+	}
+}

tstest/natlab/vmtest/vmtest.go

@@ -97,6 +97,9 @@ type Env struct {
 	selfSignedDERPCertPinning bool // serve test DERP map with sha256-raw cert pins
 	fakeACME                  bool // point tailscaled at vnet's fake ACME server
 
+	controlDNSDomain string             // MagicDNS domain for the test control server (see ControlDNS)
+	controlDNS       *tailcfg.DNSConfig // DNS config for the test control server (see ControlDNS)
+
 	// Shared resource initialization (sync.Once for things multiple nodes share).
 	vnetOnce      sync.Once
 	gokrazyOnce   sync.Once
@@ -403,6 +406,16 @@ func FakeACME() EnvOption {
 	return envOptFunc(func(e *Env) { e.fakeACME = true })
 }
 
+// ControlDNS returns an [EnvOption] that makes the test control server send
+// the given DNS configuration in MapResponses, with node names placed under
+// the given MagicDNS domain (e.g. "tailnet.test").
+func ControlDNS(magicDNSDomain string, cfg *tailcfg.DNSConfig) EnvOption {
+	return envOptFunc(func(e *Env) {
+		e.controlDNSDomain = magicDNSDomain
+		e.controlDNS = cfg
+	})
+}
+
 // AddNetwork creates a new virtual network. Arguments follow the same pattern as
 // vnet.Config.AddNetwork (string IPs, NAT types, NetworkService values).
 func (e *Env) AddNetwork(opts ...any) *vnet.Network {
@@ -430,6 +443,7 @@ type Node struct {
 	joinTailnet      bool
 	runSSH           bool // true to enable the node's Tailscale SSH server
 	noAgent          bool // true to skip TTA agent setup (e.g. macOS VMs without TTA)
+	systemdUnit      bool // true to run tailscaled via the stock systemd unit (Linux cloud VMs only)
 	advertiseRoutes  string
 	snatSubnetRoutes *bool // nil means default (true)
 	webServerPort    int
@@ -463,6 +477,8 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 			n.runSSH = true
 		case nodeOptNoAgent:
 			n.noAgent = true
+		case nodeOptSystemdUnit:
+			n.systemdUnit = true
 		case nodeOptAdvertiseRoutes:
 			n.advertiseRoutes = string(o)
 		case nodeOptSNATSubnetRoutes:
@@ -482,6 +498,10 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 		})
 	}
 
+	if n.systemdUnit && (n.os.IsGokrazy || n.os.GOOS() != "linux") {
+		e.t.Fatalf("SystemdUnit is only supported on Linux cloud VMs; node %s is %s", name, n.os.Name)
+	}
+
 	// macOS VMs require a macOS arm64 host (Apple Virtualization.framework via
 	// tailmac). Skip the test now rather than letting it proceed through the
 	// rest of the setup only to fail later.
@@ -520,6 +540,7 @@ func (n *Node) DropControlTraffic() {
 type nodeOptNoTailscale struct{}
 type nodeOptTailscaleSSH struct{}
 type nodeOptNoAgent struct{}
+type nodeOptSystemdUnit struct{}
 type nodeOptAdvertiseRoutes string
 type nodeOptSNATSubnetRoutes bool
 type nodeOptWebServer int
@@ -542,6 +563,14 @@ func TailscaleSSH() nodeOptTailscaleSSH { return nodeOptTailscaleSSH{} }
 // won't work. Useful for VMs that just need to boot and respond to ICMP.
 func NoAgent() nodeOptNoAgent { return nodeOptNoAgent{} }
 
+// SystemdUnit returns a NodeOption that makes the node run tailscaled via the
+// stock systemd unit that Linux packages ship (cmd/tailscaled/tailscaled.service
+// with cmd/tailscaled/tailscaled.defaults as its EnvironmentFile), instead of
+// launching tailscaled directly as a background process. This exercises the
+// unit's sandboxing directives and its Type=notify readiness handshake.
+// It is only supported on Linux cloud VMs (e.g. Ubuntu, Debian).
+func SystemdUnit() nodeOptSystemdUnit { return nodeOptSystemdUnit{} }
+
 // AdvertiseRoutes returns a NodeOption that configures the node to advertise
 // the given routes (comma-separated CIDRs) when joining the tailnet.
 func AdvertiseRoutes(routes string) nodeOptAdvertiseRoutes {
@@ -1642,6 +1671,11 @@ func (e *Env) initVnet() {
 		if e.selfSignedDERPCertPinning {
 			e.server.ControlServer().DERPMap = e.buildSelfSignedDERPMap()
 		}
+		if e.controlDNS != nil {
+			cs := e.server.ControlServer()
+			cs.MagicDNSDomain = e.controlDNSDomain
+			cs.DNSConfig = e.controlDNS.Clone()
+		}
 		if e.fakeACME {
 			cs := e.server.ControlServer()
 			cs.MagicDNSDomain = "tailnet.test"