diff --git a/tstest/natlab/vmtest/cloudinit.go b/tstest/natlab/vmtest/cloudinit.go index 1e0aff6ce..702ded155 100644 --- a/tstest/natlab/vmtest/cloudinit.go +++ b/tstest/natlab/vmtest/cloudinit.go @@ -123,6 +123,9 @@ func (e *Env) generateLinuxUserData(n *Node) string { } files = append(files, orFiles...) } + if n.dhcpClient == DHCPClientDhcpcd { + files = append(files, dhcpcdFiles(n)...) + } writeCloudInitFiles(&ud, files) ud.WriteString("runcmd:\n") @@ -131,6 +134,14 @@ func (e *Env) generateLinuxUserData(n *Node) string { // The debug NIC is only for SSH access from the host. ud.WriteString(" - [\"/bin/sh\", \"-c\", \"ip route del default via 10.0.2.2 dev enp0s4 2>/dev/null || true\"]\n") + // Provision the requested DNS backend before anything registers DNS + // state: dhcpcd's resolv.conf hook runs as soon as it has a lease. + writeLinuxDNSModeSetup(&ud, n.dnsMode) + + // With a non-default DHCP client, the vnet NIC has no address until that + // client runs, and the downloads below need one. + writeLinuxDHCPClientSetup(&ud, n) + // Download binaries from the files.tailscale VIP (52.52.0.6). // Use the IP directly to avoid DNS resolution issues during early boot. binDir := n.os.GOOS() + "_" + n.os.GOARCH() @@ -153,9 +164,6 @@ func (e *Env) generateLinuxUserData(n *Node) string { ud.WriteString(" - [\"sysctl\", \"-w\", \"net.ipv6.conf.all.forwarding=1\"]\n") } - // Provision the requested DNS backend before tailscaled starts. - writeLinuxDNSModeSetup(&ud, n.dnsMode) - // Start tailscaled, either via the stock systemd unit or directly in // the background. --statedir provides a VarRoot so features like // Taildrop (which needs a place to stash incoming files) have a @@ -222,6 +230,74 @@ func writeLinuxDNSModeSetup(ud *strings.Builder, mode DNSMode) { } } +// VnetNICName is the name of a Linux cloud guest's first vnet NIC. QEMU +// attaches the vnet NICs before the debug NIC, so the first lands in PCI +// slot 3; see qemu.go. The debug NIC is enp0s4. +const VnetNICName = "enp0s3" + +// dhcpcdFiles returns the files that hand the vnet NIC from systemd-networkd +// to dhcpcd. See [DHCPClientDhcpcd]. +func dhcpcdFiles(n *Node) []cloudInitFile { + return []cloudInitFile{ + { + // cloud-init writes this in its init-local stage, which systemd + // orders before network-pre.target and so before networkd starts, + // and it sorts before netplan's generated 10-netplan-*.network + // files. networkd therefore never configures this link. Leaving + // the netplan config alone is not enough: its "en*" match would + // still claim the NIC. + path: "/etc/systemd/network/05-natlab-dhcpcd.network", + content: fmt.Appendf(nil, `[Match] +MACAddress=%s +[Link] +Unmanaged=yes +`, n.vnetNode.NICMac(0)), + mode: "0644", + }, + { + // Replaces the stock file. The first five lines are the stock + // defaults that matter here. vnet has no IPv6 on these networks + // and no hostname option; noarp skips the 5s address probe. + path: "/etc/dhcpcd.conf", + content: fmt.Appendf(nil, `duid +persistent +option domain_name_servers, domain_name, domain_search +option classless_static_routes +require dhcp_server_identifier +allowinterfaces %s +ipv4only +noipv4ll +noarp +nohook hostname +`, VnetNICName), + mode: "0644", + }, + } +} + +// writeLinuxDHCPClientSetup appends the runcmd entries that start the +// requested DHCP client on the vnet NIC. It must come before anything that +// needs the NIC's address. The zero value (DHCPClientDefault) is a no-op. +func writeLinuxDHCPClientSetup(ud *strings.Builder, n *Node) { + switch n.dhcpClient { + case DHCPClientDefault: + // No additional configuration is required. + case DHCPClientDhcpcd: + // The .network drop-in selects the NIC by MAC and dhcpcd.conf by + // name. Report a mismatch on the console, where the boot log is + // collected, rather than letting the downloads below time out. + fmt.Fprintf(ud, ` - ["/bin/sh", "-c", "[ \"$(cat /sys/class/net/%[1]s/address 2>/dev/null)\" = %[2]s ] || echo natlab: %[1]s is not the vnet NIC %[2]s >&2"] +`, VnetNICName, n.vnetNode.NICMac(0)) + // dhcpcd-base ships no systemd unit. -w waits for the lease before + // returning. Its output stays on the console, which is where the + // harness looks when a node fails to come up. + fmt.Fprintf(ud, " - [\"dhcpcd\", \"-w\", \"%s\"]\n", VnetNICName) + default: + // AddNode validates the client, so an unknown value here is a bug. + panic(fmt.Sprintf("unhandled DHCPClient %q", n.dhcpClient)) + } +} + // generateFreeBSDUserData creates FreeBSD nuageinit user-data (#cloud-config) // for a node. FreeBSD's nuageinit supports a subset of cloud-init directives // including runcmd, which runs after networking is up. diff --git a/tstest/natlab/vmtest/dhcpcd_test.go b/tstest/natlab/vmtest/dhcpcd_test.go new file mode 100644 index 000000000..6571ef704 --- /dev/null +++ b/tstest/natlab/vmtest/dhcpcd_test.go @@ -0,0 +1,76 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest + +import ( + "strings" + "testing" + + "tailscale.com/tstest/natlab/vnet" +) + +// TestDhcpcdFiles checks the files [DHCPClientDhcpcd] provisions: the +// networkd drop-in matches the vnet NIC's MAC and leaves it unmanaged, and +// dhcpcd.conf restricts dhcpcd to that NIC. +func TestDhcpcdFiles(t *testing.T) { + var c vnet.Config + nw := c.AddNetwork("2.1.1.1", "192.168.1.1/24") + n := &Node{vnetNode: c.AddNode(nw)} + + files := dhcpcdFiles(n) + byPath := map[string]string{} + for _, f := range files { + byPath[f.path] = string(f.content) + } + + network, ok := byPath["/etc/systemd/network/05-natlab-dhcpcd.network"] + if !ok { + t.Fatalf("no .network file in %v", files) + } + for _, want := range []string{ + "MACAddress=" + n.vnetNode.NICMac(0).String() + "\n", + "Unmanaged=yes\n", + } { + if !strings.Contains(network, want) { + t.Errorf(".network file lacks %q:\n%s", want, network) + } + } + + conf, ok := byPath["/etc/dhcpcd.conf"] + if !ok { + t.Fatalf("no dhcpcd.conf in %v", files) + } + if want := "allowinterfaces " + VnetNICName + "\n"; !strings.Contains(conf, want) { + t.Errorf("dhcpcd.conf lacks %q:\n%s", want, conf) + } +} + +// TestWriteLinuxDHCPClientSetup checks that the default client adds no runcmd +// entries, and that dhcpcd adds a NIC check followed by a dhcpcd that waits +// for the lease. +func TestWriteLinuxDHCPClientSetup(t *testing.T) { + var c vnet.Config + nw := c.AddNetwork("2.1.1.1", "192.168.1.1/24") + n := &Node{vnetNode: c.AddNode(nw)} + + var ud strings.Builder + writeLinuxDHCPClientSetup(&ud, n) + if ud.Len() != 0 { + t.Errorf("DHCPClientDefault wrote %q, want nothing", ud.String()) + } + + n.dhcpClient = DHCPClientDhcpcd + ud.Reset() + writeLinuxDHCPClientSetup(&ud, n) + for _, want := range []string{ + "/sys/class/net/" + VnetNICName + "/address", + n.vnetNode.NICMac(0).String(), + `["dhcpcd", "-w", "` + VnetNICName + `"]`, + } { + if !strings.Contains(ud.String(), want) { + t.Errorf("DHCPClientDhcpcd wrote %q, want it to contain %s", ud.String(), want) + } + } + t.Log(ud.String()) +} diff --git a/tstest/natlab/vmtest/dns_dhcpcd_test.go b/tstest/natlab/vmtest/dns_dhcpcd_test.go new file mode 100644 index 000000000..4a9133283 --- /dev/null +++ b/tstest/natlab/vmtest/dns_dhcpcd_test.go @@ -0,0 +1,80 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest_test + +import ( + "fmt" + "slices" + "strings" + "testing" + "time" + + "tailscale.com/tstest" + "tailscale.com/tstest/natlab/vmtest" + "tailscale.com/tstest/natlab/vnet" +) + +// The snippet dhcpcd's resolv.conf hook registers for the vnet NIC's DHCP +// lease: ".". +const dhcpcdSnippet = vmtest.VnetNICName + ".dhcp" + +// newDhcpcdEnv brings up a single Ubuntu node whose vnet NIC is configured +// by dhcpcd, with openresolv installed so dhcpcd's hook registers +// dhcpcdSnippet. +func newDhcpcdEnv(t *testing.T) (*vmtest.Env, *vmtest.Node) { + t.Helper() + env := vmtest.New(t, orControlDNS) + node := env.AddNode("node", + env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT), + vmtest.OS(vmtest.Ubuntu2404), + vmtest.WithDNSMode(vmtest.DNSOpenresolv), + vmtest.WithDHCPClient(vmtest.DHCPClientDhcpcd), + ) + env.Start() + + env.AssertDNSBackend(node, "openresolv") + assertDhcpcdLease(t, env, node, vnet.FakeDNSIPv4().String()) + return env, node +} + +// assertDhcpcdLease waits for dhcpcd, not networkd, to hold the vnet NIC's +// lease, with its hook having registered dhcpcdSnippet naming the given +// nameserver. The hook runs after dhcpcd has bound the address, so this +// may briefly lag a `dhcpcd -w` that has already returned. +func assertDhcpcdLease(t *testing.T, env *vmtest.Env, n *vmtest.Node, nameserver string) { + t.Helper() + const cmd = "networkctl status " + vmtest.VnetNICName + " | grep -m1 State:; resolvconf -i; resolvconf -l " + dhcpcdSnippet + var last string + if err := tstest.WaitFor(15*time.Second, func() error { + out, err := env.SSHExec(n, cmd) + last = out + if err != nil { + return fmt.Errorf("%s: %v (%s)", cmd, err, strings.TrimSpace(out)) + } + if !strings.Contains(out, "unmanaged") { + return fmt.Errorf("networkd still manages %s", vmtest.VnetNICName) + } + if !slices.Contains(strings.Fields(out), dhcpcdSnippet) { + return fmt.Errorf("resolvconf has no %s snippet", dhcpcdSnippet) + } + if !strings.Contains(out, "nameserver "+nameserver) { + return fmt.Errorf("%s snippet does not name %s", dhcpcdSnippet, nameserver) + } + return nil + }); err != nil { + t.Fatalf("DHCP lease check failed: %v\nOutput:\n%s", err, last) + } +} + +// TestDhcpcdOpenresolvDNS checks the dhcpcd provisioning itself: dhcpcd holds +// the lease, its hook registered the lease's nameserver with openresolv, and +// tailscaled forwards public names there. +func TestDhcpcdOpenresolvDNS(t *testing.T) { + env, node := newDhcpcdEnv(t) + assertOpenresolvResolvConf(t, env, node, + []string{orSignature, orQuad100}, + []string{vnet.FakeDNSIPv4().String()}, + ) + assertResolves(t, env, node, orUpstreamOnlyName, orUpstreamOnlyIP) +} diff --git a/tstest/natlab/vmtest/dns_openresolv_test.go b/tstest/natlab/vmtest/dns_openresolv_test.go index 7360f4d63..350890e55 100644 --- a/tstest/natlab/vmtest/dns_openresolv_test.go +++ b/tstest/natlab/vmtest/dns_openresolv_test.go @@ -49,22 +49,24 @@ const ( orQuad100 = "100.100.100.100" ) +// orControlDNS is the control DNS config the openresolv tests share. +var orControlDNS = vmtest.ControlDNS(orMagicDNSDomain, &tailcfg.DNSConfig{ + Proxied: true, // MagicDNS, so there's a route and quad-100 is in play + Domains: []string{orLocalDomain}, + Routes: map[string][]*dnstype.Resolver{ + orLocalDomain: nil, // answer locally, from ExtraRecords + }, + ExtraRecords: []tailcfg.DNSRecord{ + {Name: orLocalName, Type: "A", Value: orLocalIP}, + }, +}) + // newOpenresolvEnv brings up a single Ubuntu node running upstream openresolv // with an existing but empty snippet directory, so openresolv has no snippets // registered until something adds one. func newOpenresolvEnv(t *testing.T) (*vmtest.Env, *vmtest.Node) { t.Helper() - env := vmtest.New(t, - vmtest.ControlDNS(orMagicDNSDomain, &tailcfg.DNSConfig{ - Proxied: true, // MagicDNS, so there's a route and quad-100 is in play - Domains: []string{orLocalDomain}, - Routes: map[string][]*dnstype.Resolver{ - orLocalDomain: nil, // answer locally, from ExtraRecords - }, - ExtraRecords: []tailcfg.DNSRecord{ - {Name: orLocalName, Type: "A", Value: orLocalIP}, - }, - })) + env := vmtest.New(t, orControlDNS) node := env.AddNode("node", env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT), vmtest.OS(vmtest.Ubuntu2404), diff --git a/tstest/natlab/vmtest/vmtest.go b/tstest/natlab/vmtest/vmtest.go index 3a94f7483..fe1719e4e 100644 --- a/tstest/natlab/vmtest/vmtest.go +++ b/tstest/natlab/vmtest/vmtest.go @@ -469,8 +469,9 @@ type Node struct { advertiseRoutes string snatSubnetRoutes *bool // nil means default (true) webServerPort int - sshPort int // host port for SSH debug access (cloud VMs only) - dnsMode DNSMode // desired Linux DNS backend to provision; "" means the image default + sshPort int // host port for SSH debug access (cloud VMs only) + dnsMode DNSMode // desired Linux DNS backend to provision; "" means the image default + dhcpClient DHCPClient // DHCP client for the vnet NIC; "" means the image default } // AddNode creates a new VM node. The name is used for identification and as the @@ -516,11 +517,28 @@ func (e *Env) AddNode(name string, opts ...any) *Node { e.t.Fatalf("AddNode(%q): unsupported DNSMode %q", name, DNSMode(o)) } n.dnsMode = DNSMode(o) + case nodeOptDHCPClient: + switch c := DHCPClient(o); c { + case DHCPClientDefault, DHCPClientDhcpcd: + n.dhcpClient = c + default: + e.t.Fatalf("AddNode(%q): unsupported DHCPClient %q", name, c) + } default: // Pass through to vnet (TailscaledEnv, NodeOption, MAC, etc.) vnetOpts = append(vnetOpts, o) } } + if n.dhcpClient == DHCPClientDhcpcd { + // Only this image is known to ship dhcpcd-base, and the dhcpcd.conf + // written by dhcpcdFiles names the single vnet NIC. + if n.os.Name != Ubuntu2404.Name { + e.t.Fatalf("AddNode(%q): DHCPClientDhcpcd requires the %s image, got %s", name, Ubuntu2404.Name, n.os.Name) + } + if len(n.nets) != 1 { + e.t.Fatalf("AddNode(%q): DHCPClientDhcpcd requires exactly one network, got %d", name, len(n.nets)) + } + } if e.fakeACME { vnetOpts = append(vnetOpts, vnet.TailscaledEnv{ Key: "TS_DEBUG_ACME_DIRECTORY_URL", @@ -591,6 +609,29 @@ type nodeOptAdvertiseRoutes string type nodeOptSNATSubnetRoutes bool type nodeOptWebServer int type nodeOptDNSMode DNSMode +type nodeOptDHCPClient DHCPClient + +// DHCPClient says which DHCP client configures the guest's vnet NIC. +type DHCPClient string + +const ( + // DHCPClientDefault leaves the image's networking alone, so the NIC is + // configured by whatever the image runs by default (systemd-networkd on + // the cloud images). + DHCPClientDefault DHCPClient = "" + + // DHCPClientDhcpcd makes systemd-networkd leave the vnet NIC unmanaged and + // runs the dhcpcd that the Ubuntu 24.04 image ships (dhcpcd-base) on it + // instead. dhcpcd configures the lease's address and routes itself, then + // runs its hook scripts. The resolv.conf hook calls resolvconf when one + // is installed, registering a snippet named ".dhcp", and + // writes /etc/resolv.conf directly otherwise. Combine with + // [DNSOpenresolv] so the hook registers the snippet instead of writing + // resolv.conf. + // + // Only supported on [Ubuntu2404] nodes with a single network. + DHCPClientDhcpcd DHCPClient = "dhcpcd" +) // DNSMode is a provisioning directive, not a DNS-backend name: it says what, if // anything, to do to the guest's DNS before tailscaled starts, letting one @@ -614,10 +655,12 @@ const ( // DNSOpenresolv masks systemd-resolved and installs upstream openresolv // (which no cloud image ships), so tailscaled selects the "openresolv" - // manager. Its key directory is created empty, so the only snippet ever - // registered is Tailscale's own. Before the fix for tailscale/tailscale#20825, - // tailscaled handled that state wrong. openresolv reports "no snippets" by - // exiting 2, and net/dns treated that non-zero exit as a hard failure. + // manager. Its key directory is created empty, so unless [WithDHCPClient] + // selects a client whose resolv.conf hook registers a snippet, the only + // one ever registered is Tailscale's own. Before the fix for + // tailscale/tailscale#20825, tailscaled handled an empty key directory + // wrong: openresolv reports "no snippets" by exiting 2, and net/dns + // treated that non-zero exit as a hard failure. // // openresolv's sources are vendored into the tree; see openresolv.go. DNSOpenresolv DNSMode = "openresolv" @@ -670,6 +713,10 @@ func WebServer(port int) nodeOptWebServer { return nodeOptWebServer(port) } // images; ignored for gokrazy/macOS. See [DNSMode]. func WithDNSMode(m DNSMode) nodeOptDNSMode { return nodeOptDNSMode(m) } +// WithDHCPClient returns a NodeOption that provisions the node so the given +// DHCP client configures its vnet NIC. See [DHCPClient]. +func WithDHCPClient(c DHCPClient) nodeOptDHCPClient { return nodeOptDHCPClient(c) } + // Start initializes the virtual network, boots all VMs in parallel, and waits // for all TTA agents to connect. It should be called after all AddNetwork/AddNode calls. func (e *Env) Start() {