tstest/natlab/vmtest: add a dhcpcd DHCP client option for Ubuntu guests (#21672)

WithDHCPClient(DHCPClientDhcpcd) makes systemd-networkd leave the vnet NIC
unmanaged and runs the dhcpcd that the Ubuntu 24.04 image ships on it.
dhcpcd is the DHCP client on Raspberry Pi OS and other distributions, and
unlike networkd it configures the interface first and runs its resolvconf
hook afterwards. TestDhcpcdOpenresolvDNS checks that dhcpcd holds the
lease, its hook registered the lease's nameserver with openresolv, and
tailscaled forwards through it.

Updates #21607

Signed-off-by: Brendan Creane <bcreane@gmail.com>

tstest/natlab/vmtest/cloudinit.go

@@ -123,6 +123,9 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 		}
 		files = append(files, orFiles...)
 	}
+	if n.dhcpClient == DHCPClientDhcpcd {
+		files = append(files, dhcpcdFiles(n)...)
+	}
 	writeCloudInitFiles(&ud, files)
 
 	ud.WriteString("runcmd:\n")
@@ -131,6 +134,14 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 	// The debug NIC is only for SSH access from the host.
 	ud.WriteString("  - [\"/bin/sh\", \"-c\", \"ip route del default via 10.0.2.2 dev enp0s4 2>/dev/null || true\"]\n")
 
+	// Provision the requested DNS backend before anything registers DNS
+	// state: dhcpcd's resolv.conf hook runs as soon as it has a lease.
+	writeLinuxDNSModeSetup(&ud, n.dnsMode)
+
+	// With a non-default DHCP client, the vnet NIC has no address until that
+	// client runs, and the downloads below need one.
+	writeLinuxDHCPClientSetup(&ud, n)
+
 	// Download binaries from the files.tailscale VIP (52.52.0.6).
 	// Use the IP directly to avoid DNS resolution issues during early boot.
 	binDir := n.os.GOOS() + "_" + n.os.GOARCH()
@@ -153,9 +164,6 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 		ud.WriteString("  - [\"sysctl\", \"-w\", \"net.ipv6.conf.all.forwarding=1\"]\n")
 	}
 
-	// Provision the requested DNS backend before tailscaled starts.
-	writeLinuxDNSModeSetup(&ud, n.dnsMode)
-
 	// Start tailscaled, either via the stock systemd unit or directly in
 	// the background. --statedir provides a VarRoot so features like
 	// Taildrop (which needs a place to stash incoming files) have a
@@ -222,6 +230,74 @@ func writeLinuxDNSModeSetup(ud *strings.Builder, mode DNSMode) {
 	}
 }
 
+// VnetNICName is the name of a Linux cloud guest's first vnet NIC. QEMU
+// attaches the vnet NICs before the debug NIC, so the first lands in PCI
+// slot 3; see qemu.go. The debug NIC is enp0s4.
+const VnetNICName = "enp0s3"
+
+// dhcpcdFiles returns the files that hand the vnet NIC from systemd-networkd
+// to dhcpcd. See [DHCPClientDhcpcd].
+func dhcpcdFiles(n *Node) []cloudInitFile {
+	return []cloudInitFile{
+		{
+			// cloud-init writes this in its init-local stage, which systemd
+			// orders before network-pre.target and so before networkd starts,
+			// and it sorts before netplan's generated 10-netplan-*.network
+			// files. networkd therefore never configures this link. Leaving
+			// the netplan config alone is not enough: its "en*" match would
+			// still claim the NIC.
+			path: "/etc/systemd/network/05-natlab-dhcpcd.network",
+			content: fmt.Appendf(nil, `[Match]
+MACAddress=%s
+[Link]
+Unmanaged=yes
+`, n.vnetNode.NICMac(0)),
+			mode: "0644",
+		},
+		{
+			// Replaces the stock file. The first five lines are the stock
+			// defaults that matter here. vnet has no IPv6 on these networks
+			// and no hostname option; noarp skips the 5s address probe.
+			path: "/etc/dhcpcd.conf",
+			content: fmt.Appendf(nil, `duid
+persistent
+option domain_name_servers, domain_name, domain_search
+option classless_static_routes
+require dhcp_server_identifier
+allowinterfaces %s
+ipv4only
+noipv4ll
+noarp
+nohook hostname
+`, VnetNICName),
+			mode: "0644",
+		},
+	}
+}
+
+// writeLinuxDHCPClientSetup appends the runcmd entries that start the
+// requested DHCP client on the vnet NIC. It must come before anything that
+// needs the NIC's address. The zero value (DHCPClientDefault) is a no-op.
+func writeLinuxDHCPClientSetup(ud *strings.Builder, n *Node) {
+	switch n.dhcpClient {
+	case DHCPClientDefault:
+		// No additional configuration is required.
+	case DHCPClientDhcpcd:
+		// The .network drop-in selects the NIC by MAC and dhcpcd.conf by
+		// name. Report a mismatch on the console, where the boot log is
+		// collected, rather than letting the downloads below time out.
+		fmt.Fprintf(ud, `  - ["/bin/sh", "-c", "[ \"$(cat /sys/class/net/%[1]s/address 2>/dev/null)\" = %[2]s ] || echo natlab: %[1]s is not the vnet NIC %[2]s >&2"]
+`, VnetNICName, n.vnetNode.NICMac(0))
+		// dhcpcd-base ships no systemd unit. -w waits for the lease before
+		// returning. Its output stays on the console, which is where the
+		// harness looks when a node fails to come up.
+		fmt.Fprintf(ud, "  - [\"dhcpcd\", \"-w\", \"%s\"]\n", VnetNICName)
+	default:
+		// AddNode validates the client, so an unknown value here is a bug.
+		panic(fmt.Sprintf("unhandled DHCPClient %q", n.dhcpClient))
+	}
+}
+
 // generateFreeBSDUserData creates FreeBSD nuageinit user-data (#cloud-config)
 // for a node. FreeBSD's nuageinit supports a subset of cloud-init directives
 // including runcmd, which runs after networking is up.

tstest/natlab/vmtest/dhcpcd_test.go

@@ -0,0 +1,76 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest
+
+import (
+	"strings"
+	"testing"
+
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+// TestDhcpcdFiles checks the files [DHCPClientDhcpcd] provisions: the
+// networkd drop-in matches the vnet NIC's MAC and leaves it unmanaged, and
+// dhcpcd.conf restricts dhcpcd to that NIC.
+func TestDhcpcdFiles(t *testing.T) {
+	var c vnet.Config
+	nw := c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+	n := &Node{vnetNode: c.AddNode(nw)}
+
+	files := dhcpcdFiles(n)
+	byPath := map[string]string{}
+	for _, f := range files {
+		byPath[f.path] = string(f.content)
+	}
+
+	network, ok := byPath["/etc/systemd/network/05-natlab-dhcpcd.network"]
+	if !ok {
+		t.Fatalf("no .network file in %v", files)
+	}
+	for _, want := range []string{
+		"MACAddress=" + n.vnetNode.NICMac(0).String() + "\n",
+		"Unmanaged=yes\n",
+	} {
+		if !strings.Contains(network, want) {
+			t.Errorf(".network file lacks %q:\n%s", want, network)
+		}
+	}
+
+	conf, ok := byPath["/etc/dhcpcd.conf"]
+	if !ok {
+		t.Fatalf("no dhcpcd.conf in %v", files)
+	}
+	if want := "allowinterfaces " + VnetNICName + "\n"; !strings.Contains(conf, want) {
+		t.Errorf("dhcpcd.conf lacks %q:\n%s", want, conf)
+	}
+}
+
+// TestWriteLinuxDHCPClientSetup checks that the default client adds no runcmd
+// entries, and that dhcpcd adds a NIC check followed by a dhcpcd that waits
+// for the lease.
+func TestWriteLinuxDHCPClientSetup(t *testing.T) {
+	var c vnet.Config
+	nw := c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+	n := &Node{vnetNode: c.AddNode(nw)}
+
+	var ud strings.Builder
+	writeLinuxDHCPClientSetup(&ud, n)
+	if ud.Len() != 0 {
+		t.Errorf("DHCPClientDefault wrote %q, want nothing", ud.String())
+	}
+
+	n.dhcpClient = DHCPClientDhcpcd
+	ud.Reset()
+	writeLinuxDHCPClientSetup(&ud, n)
+	for _, want := range []string{
+		"/sys/class/net/" + VnetNICName + "/address",
+		n.vnetNode.NICMac(0).String(),
+		`["dhcpcd", "-w", "` + VnetNICName + `"]`,
+	} {
+		if !strings.Contains(ud.String(), want) {
+			t.Errorf("DHCPClientDhcpcd wrote %q, want it to contain %s", ud.String(), want)
+		}
+	}
+	t.Log(ud.String())
+}

tstest/natlab/vmtest/dns_dhcpcd_test.go

@@ -0,0 +1,80 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"tailscale.com/tstest"
+	"tailscale.com/tstest/natlab/vmtest"
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+// The snippet dhcpcd's resolv.conf hook registers for the vnet NIC's DHCP
+// lease: "<interface>.<protocol>".
+const dhcpcdSnippet = vmtest.VnetNICName + ".dhcp"
+
+// newDhcpcdEnv brings up a single Ubuntu node whose vnet NIC is configured
+// by dhcpcd, with openresolv installed so dhcpcd's hook registers
+// dhcpcdSnippet.
+func newDhcpcdEnv(t *testing.T) (*vmtest.Env, *vmtest.Node) {
+	t.Helper()
+	env := vmtest.New(t, orControlDNS)
+	node := env.AddNode("node",
+		env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT),
+		vmtest.OS(vmtest.Ubuntu2404),
+		vmtest.WithDNSMode(vmtest.DNSOpenresolv),
+		vmtest.WithDHCPClient(vmtest.DHCPClientDhcpcd),
+	)
+	env.Start()
+
+	env.AssertDNSBackend(node, "openresolv")
+	assertDhcpcdLease(t, env, node, vnet.FakeDNSIPv4().String())
+	return env, node
+}
+
+// assertDhcpcdLease waits for dhcpcd, not networkd, to hold the vnet NIC's
+// lease, with its hook having registered dhcpcdSnippet naming the given
+// nameserver. The hook runs after dhcpcd has bound the address, so this
+// may briefly lag a `dhcpcd -w` that has already returned.
+func assertDhcpcdLease(t *testing.T, env *vmtest.Env, n *vmtest.Node, nameserver string) {
+	t.Helper()
+	const cmd = "networkctl status " + vmtest.VnetNICName + " | grep -m1 State:; resolvconf -i; resolvconf -l " + dhcpcdSnippet
+	var last string
+	if err := tstest.WaitFor(15*time.Second, func() error {
+		out, err := env.SSHExec(n, cmd)
+		last = out
+		if err != nil {
+			return fmt.Errorf("%s: %v (%s)", cmd, err, strings.TrimSpace(out))
+		}
+		if !strings.Contains(out, "unmanaged") {
+			return fmt.Errorf("networkd still manages %s", vmtest.VnetNICName)
+		}
+		if !slices.Contains(strings.Fields(out), dhcpcdSnippet) {
+			return fmt.Errorf("resolvconf has no %s snippet", dhcpcdSnippet)
+		}
+		if !strings.Contains(out, "nameserver "+nameserver) {
+			return fmt.Errorf("%s snippet does not name %s", dhcpcdSnippet, nameserver)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("DHCP lease check failed: %v\nOutput:\n%s", err, last)
+	}
+}
+
+// TestDhcpcdOpenresolvDNS checks the dhcpcd provisioning itself: dhcpcd holds
+// the lease, its hook registered the lease's nameserver with openresolv, and
+// tailscaled forwards public names there.
+func TestDhcpcdOpenresolvDNS(t *testing.T) {
+	env, node := newDhcpcdEnv(t)
+	assertOpenresolvResolvConf(t, env, node,
+		[]string{orSignature, orQuad100},
+		[]string{vnet.FakeDNSIPv4().String()},
+	)
+	assertResolves(t, env, node, orUpstreamOnlyName, orUpstreamOnlyIP)
+}

tstest/natlab/vmtest/dns_openresolv_test.go

@@ -49,22 +49,24 @@ const (
 	orQuad100 = "100.100.100.100"
 )
 
+// orControlDNS is the control DNS config the openresolv tests share.
+var orControlDNS = vmtest.ControlDNS(orMagicDNSDomain, &tailcfg.DNSConfig{
+	Proxied: true, // MagicDNS, so there's a route and quad-100 is in play
+	Domains: []string{orLocalDomain},
+	Routes: map[string][]*dnstype.Resolver{
+		orLocalDomain: nil, // answer locally, from ExtraRecords
+	},
+	ExtraRecords: []tailcfg.DNSRecord{
+		{Name: orLocalName, Type: "A", Value: orLocalIP},
+	},
+})
+
 // newOpenresolvEnv brings up a single Ubuntu node running upstream openresolv
 // with an existing but empty snippet directory, so openresolv has no snippets
 // registered until something adds one.
 func newOpenresolvEnv(t *testing.T) (*vmtest.Env, *vmtest.Node) {
 	t.Helper()
-	env := vmtest.New(t,
-		vmtest.ControlDNS(orMagicDNSDomain, &tailcfg.DNSConfig{
-			Proxied: true, // MagicDNS, so there's a route and quad-100 is in play
-			Domains: []string{orLocalDomain},
-			Routes: map[string][]*dnstype.Resolver{
-				orLocalDomain: nil, // answer locally, from ExtraRecords
-			},
-			ExtraRecords: []tailcfg.DNSRecord{
-				{Name: orLocalName, Type: "A", Value: orLocalIP},
-			},
-		}))
+	env := vmtest.New(t, orControlDNS)
 	node := env.AddNode("node",
 		env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT),
 		vmtest.OS(vmtest.Ubuntu2404),

tstest/natlab/vmtest/vmtest.go

@@ -469,8 +469,9 @@ type Node struct {
 	advertiseRoutes  string
 	snatSubnetRoutes *bool // nil means default (true)
 	webServerPort    int
-	sshPort          int     // host port for SSH debug access (cloud VMs only)
-	dnsMode          DNSMode // desired Linux DNS backend to provision; "" means the image default
+	sshPort          int        // host port for SSH debug access (cloud VMs only)
+	dnsMode          DNSMode    // desired Linux DNS backend to provision; "" means the image default
+	dhcpClient       DHCPClient // DHCP client for the vnet NIC; "" means the image default
 }
 
 // AddNode creates a new VM node. The name is used for identification and as the
@@ -516,11 +517,28 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 				e.t.Fatalf("AddNode(%q): unsupported DNSMode %q", name, DNSMode(o))
 			}
 			n.dnsMode = DNSMode(o)
+		case nodeOptDHCPClient:
+			switch c := DHCPClient(o); c {
+			case DHCPClientDefault, DHCPClientDhcpcd:
+				n.dhcpClient = c
+			default:
+				e.t.Fatalf("AddNode(%q): unsupported DHCPClient %q", name, c)
+			}
 		default:
 			// Pass through to vnet (TailscaledEnv, NodeOption, MAC, etc.)
 			vnetOpts = append(vnetOpts, o)
 		}
 	}
+	if n.dhcpClient == DHCPClientDhcpcd {
+		// Only this image is known to ship dhcpcd-base, and the dhcpcd.conf
+		// written by dhcpcdFiles names the single vnet NIC.
+		if n.os.Name != Ubuntu2404.Name {
+			e.t.Fatalf("AddNode(%q): DHCPClientDhcpcd requires the %s image, got %s", name, Ubuntu2404.Name, n.os.Name)
+		}
+		if len(n.nets) != 1 {
+			e.t.Fatalf("AddNode(%q): DHCPClientDhcpcd requires exactly one network, got %d", name, len(n.nets))
+		}
+	}
 	if e.fakeACME {
 		vnetOpts = append(vnetOpts, vnet.TailscaledEnv{
 			Key:   "TS_DEBUG_ACME_DIRECTORY_URL",
@@ -591,6 +609,29 @@ type nodeOptAdvertiseRoutes string
 type nodeOptSNATSubnetRoutes bool
 type nodeOptWebServer int
 type nodeOptDNSMode DNSMode
+type nodeOptDHCPClient DHCPClient
+
+// DHCPClient says which DHCP client configures the guest's vnet NIC.
+type DHCPClient string
+
+const (
+	// DHCPClientDefault leaves the image's networking alone, so the NIC is
+	// configured by whatever the image runs by default (systemd-networkd on
+	// the cloud images).
+	DHCPClientDefault DHCPClient = ""
+
+	// DHCPClientDhcpcd makes systemd-networkd leave the vnet NIC unmanaged and
+	// runs the dhcpcd that the Ubuntu 24.04 image ships (dhcpcd-base) on it
+	// instead. dhcpcd configures the lease's address and routes itself, then
+	// runs its hook scripts. The resolv.conf hook calls resolvconf when one
+	// is installed, registering a snippet named "<interface>.dhcp", and
+	// writes /etc/resolv.conf directly otherwise. Combine with
+	// [DNSOpenresolv] so the hook registers the snippet instead of writing
+	// resolv.conf.
+	//
+	// Only supported on [Ubuntu2404] nodes with a single network.
+	DHCPClientDhcpcd DHCPClient = "dhcpcd"
+)
 
 // DNSMode is a provisioning directive, not a DNS-backend name: it says what, if
 // anything, to do to the guest's DNS before tailscaled starts, letting one
@@ -614,10 +655,12 @@ const (
 
 	// DNSOpenresolv masks systemd-resolved and installs upstream openresolv
 	// (which no cloud image ships), so tailscaled selects the "openresolv"
-	// manager. Its key directory is created empty, so the only snippet ever
-	// registered is Tailscale's own. Before the fix for tailscale/tailscale#20825,
-	// tailscaled handled that state wrong. openresolv reports "no snippets" by
-	// exiting 2, and net/dns treated that non-zero exit as a hard failure.
+	// manager. Its key directory is created empty, so unless [WithDHCPClient]
+	// selects a client whose resolv.conf hook registers a snippet, the only
+	// one ever registered is Tailscale's own. Before the fix for
+	// tailscale/tailscale#20825, tailscaled handled an empty key directory
+	// wrong: openresolv reports "no snippets" by exiting 2, and net/dns
+	// treated that non-zero exit as a hard failure.
 	//
 	// openresolv's sources are vendored into the tree; see openresolv.go.
 	DNSOpenresolv DNSMode = "openresolv"
@@ -670,6 +713,10 @@ func WebServer(port int) nodeOptWebServer { return nodeOptWebServer(port) }
 // images; ignored for gokrazy/macOS. See [DNSMode].
 func WithDNSMode(m DNSMode) nodeOptDNSMode { return nodeOptDNSMode(m) }
 
+// WithDHCPClient returns a NodeOption that provisions the node so the given
+// DHCP client configures its vnet NIC. See [DHCPClient].
+func WithDHCPClient(c DHCPClient) nodeOptDHCPClient { return nodeOptDHCPClient(c) }
+
 // Start initializes the virtual network, boots all VMs in parallel, and waits
 // for all TTA agents to connect. It should be called after all AddNetwork/AddNode calls.
 func (e *Env) Start() {