tstest/natlab/vmtest: add VM coverage for openresolv DNS backend (#20999)

* net/dns: export OSConfigurationReadWarnable

A natlab vmtest checks that a node is not reporting this warning. Exporting
the Warnable lets the test take the warning's text from it, instead of
keeping a copy of the wording that could stop matching without failing.

Updates #20825
Updates tailscale/corp#44793

Signed-off-by: Brendan Creane <bcreane@gmail.com>

* tstest/natlab/vmtest: add VM coverage for the openresolv DNS backend

dnsMode() picks one of five Linux DNS backends, and natlab could provision
only systemd-resolved and direct. Add a DNSOpenresolv mode and a VM test for
it, so the backend behind #20825 is covered.

No cloud image ships openresolv and a guest cannot download it, so its two
source files are vendored under testdata and installed with cloud-init's
write_files. openresolv's build is a set of sed substitutions with nothing to
compile, so resolvconf.go does the substitutions in process.

Updates #20825
Updates tailscale/corp#44793

Signed-off-by: Brendan Creane <bcreane@gmail.com>

---------

Signed-off-by: Brendan Creane <bcreane@gmail.com>

net/dns/manager.go

@@ -282,7 +282,10 @@ func compileHostEntries(cfg Config) (hosts []*HostEntry) {
 	return hosts
 }
 
-var osConfigurationReadWarnable = health.Register(&health.Warnable{
+// OSConfigurationReadWarnable is a Warnable set when Tailscale cannot read the
+// DNS configuration the OS was using before Tailscale took over. It is
+// exported so that a test can name it rather than repeat its wording.
+var OSConfigurationReadWarnable = health.Register(&health.Warnable{
 	Code:  "dns-read-os-config-failed",
 	Title: "Failed to read system DNS configuration",
 	Text: func(args health.Args) string {
@@ -436,14 +439,14 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig
 			// config instead of erroring and leaving the old OS config.
 			// Sandboxed macOS is excluded: it does have a base config
 			// (/etc/resolv.conf), so this error is a real read failure there.
-			m.health.SetHealthy(osConfigurationReadWarnable)
+			m.health.SetHealthy(OSConfigurationReadWarnable)
 			ocfg.MatchDomains = cfg.matchDomains()
 			return rcfg, ocfg, nil
 		}
-		m.health.SetUnhealthy(osConfigurationReadWarnable, health.Args{health.ArgError: err.Error()})
+		m.health.SetUnhealthy(OSConfigurationReadWarnable, health.Args{health.ArgError: err.Error()})
 		return resolver.Config{}, OSConfig{}, err
 	}
-	m.health.SetHealthy(osConfigurationReadWarnable)
+	m.health.SetHealthy(OSConfigurationReadWarnable)
 
 	// On iOS only (for now), check if all route names point to resources inside the tailnet.
 	// If so, we can set those names as MatchDomains to enable a split DNS configuration

tstest/natlab/vmtest/cloudinit.go

@@ -112,9 +112,18 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 		ud.WriteString(fmt.Sprintf("    ssh_authorized_keys:\n      - %s\n", strings.TrimSpace(string(pubkey))))
 	}
 
+	var files []cloudInitFile
 	if n.systemdUnit {
-		e.writeSystemdUnitFiles(&ud, n)
+		files = append(files, e.systemdUnitFiles(n)...)
 	}
+	if n.dnsMode == DNSOpenresolv {
+		orFiles, err := buildOpenresolv()
+		if err != nil {
+			e.t.Fatalf("building openresolv for the guest: %v", err)
+		}
+		files = append(files, orFiles...)
+	}
+	writeCloudInitFiles(&ud, files)
 
 	ud.WriteString("runcmd:\n")
 
@@ -190,6 +199,26 @@ func writeLinuxDNSModeSetup(ud *strings.Builder, mode DNSMode) {
 		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"systemctl disable --now systemd-resolved 2>/dev/null || true\"]\n")
 		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"systemctl mask systemd-resolved 2>/dev/null || true\"]\n")
 		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"rm -f /etc/resolv.conf && printf 'nameserver %s\\\\n' >/etc/resolv.conf\"]\n", vnet.FakeDNSIPv4())
+	case DNSOpenresolv:
+		// Mask systemd-resolved, so that resolvconfStyle() in
+		// net/dns/resolvconf.go finds a resolvconf that is not Debian's.
+		// dnsMode() then selects the "openresolv" manager. openresolv's own
+		// files are installed by write_files; see buildOpenresolv.
+		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"systemctl disable --now systemd-resolved 2>/dev/null || true\"]\n")
+		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"systemctl mask systemd-resolved 2>/dev/null || true\"]\n")
+		// Create resolvconf's key directory, empty. Left alone, resolvconf
+		// would not create it until the first snippet was registered, and a
+		// missing directory makes "resolvconf -l" exit 0 instead of 2. Only an
+		// existing empty directory produces the state [DNSOpenresolv]
+		// describes. write_files cannot create this one, because it creates a
+		// directory only as a side effect of writing a file into it.
+		fmt.Fprintf(ud, "  - [\"mkdir\", \"-p\", \"%s\"]\n", openresolvKeyDir)
+		// The signature line is required verbatim: openresolv's libc
+		// subscriber refuses to overwrite a resolv.conf that lacks it, and
+		// resolvOwner() in net/dns/direct.go needs "resolvconf" in the leading
+		// comments for dnsMode() to take its resolvconf branch. The nameserver
+		// keeps the guest resolving until tailscaled takes over.
+		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"rm -f /etc/resolv.conf && printf '# Generated by resolvconf\\\\nnameserver %s\\\\n' >/etc/resolv.conf\"]\n", vnet.FakeDNSIPv4())
 	}
 }
 
@@ -272,12 +301,39 @@ func (e *Env) generateFreeBSDUserData(n *Node) string {
 	return ud.String()
 }
 
-// writeSystemdUnitFiles appends a cloud-init write_files section that
-// installs the stock tailscaled systemd unit from the source tree, along
-// with the packaging's /etc/default/tailscaled EnvironmentFile (plus any
-// per-node TailscaledEnv variables). File contents are base64-encoded to
-// sidestep YAML quoting.
-func (e *Env) writeSystemdUnitFiles(ud *strings.Builder, n *Node) {
+// cloudInitFile is one file cloud-init writes into the guest. cloud-init
+// writes them during its config stage, which runs before the runcmd entries,
+// and it creates any missing parent directories.
+type cloudInitFile struct {
+	path    string
+	content []byte
+	mode    string // octal permissions, e.g. "0755"; empty leaves cloud-init's default
+}
+
+// writeCloudInitFiles appends the write_files section. There can be only one
+// such section per user-data document, so every file a node needs has to be
+// collected before this is called.
+//
+// Contents are base64-encoded to sidestep YAML quoting.
+func writeCloudInitFiles(ud *strings.Builder, files []cloudInitFile) {
+	if len(files) == 0 {
+		return
+	}
+	ud.WriteString("write_files:\n")
+	for _, f := range files {
+		fmt.Fprintf(ud, "  - path: %s\n", f.path)
+		fmt.Fprintf(ud, "    encoding: b64\n")
+		if f.mode != "" {
+			fmt.Fprintf(ud, "    permissions: '%s'\n", f.mode)
+		}
+		fmt.Fprintf(ud, "    content: %s\n", base64.StdEncoding.EncodeToString(f.content))
+	}
+}
+
+// systemdUnitFiles returns the stock tailscaled systemd unit from the source
+// tree, along with the packaging's /etc/default/tailscaled EnvironmentFile
+// (plus any per-node TailscaledEnv variables).
+func (e *Env) systemdUnitFiles(n *Node) []cloudInitFile {
 	modRoot, err := findModRoot()
 	if err != nil {
 		e.t.Fatalf("finding module root for tailscaled.service: %v", err)
@@ -296,14 +352,10 @@ func (e *Env) writeSystemdUnitFiles(ud *strings.Builder, n *Node) {
 		fmt.Fprintf(&envFile, "%s=%q\n", env.Key, env.Value)
 	}
 
-	ud.WriteString("write_files:\n")
-	writeFile := func(path string, content []byte) {
-		fmt.Fprintf(ud, "  - path: %s\n", path)
-		fmt.Fprintf(ud, "    encoding: b64\n")
-		fmt.Fprintf(ud, "    content: %s\n", base64.StdEncoding.EncodeToString(content))
+	return []cloudInitFile{
+		{path: "/etc/systemd/system/tailscaled.service", content: unit},
+		{path: "/etc/default/tailscaled", content: []byte(envFile.String())},
 	}
-	writeFile("/etc/systemd/system/tailscaled.service", unit)
-	writeFile("/etc/default/tailscaled", []byte(envFile.String()))
 }
 
 func tailscaledEnvPrefix(n *Node) string {

tstest/natlab/vmtest/dns_openresolv_test.go

@@ -0,0 +1,213 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"tailscale.com/cmd/tailscale/tsdnsjsonv0"
+	"tailscale.com/health"
+	"tailscale.com/net/dns"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstest"
+	"tailscale.com/tstest/natlab/vmtest"
+	"tailscale.com/tstest/natlab/vnet"
+	"tailscale.com/types/dnstype"
+)
+
+// The openresolv tests use a routed domain that is answered locally from an
+// extra record. That is the cheapest DNS config that still reaches
+// GetBaseConfig. It has no default resolvers, which would short-circuit
+// compileConfig. It also takes no split-DNS shortcut, because openresolv
+// reports SupportsSplitDNS() == false.
+const (
+	orMagicDNSDomain = "tailnet.test"
+	orLocalDomain    = "local.example"
+	orLocalName      = "host." + orLocalDomain
+
+	// This address is outside the 100.64.x.y block testcontrol assigns to
+	// nodes, so an answer can only have come from the extra record.
+	orLocalIP = "100.99.99.99"
+
+	// The signature line openresolv writes at the top of resolv.conf.
+	orSignature = "# Generated by resolvconf"
+
+	// Tailscale's own resolver. tailscaled points resolv.conf at it once it has
+	// configured DNS.
+	orQuad100 = "100.100.100.100"
+)
+
+// newOpenresolvEnv brings up a single Ubuntu node running upstream openresolv
+// with an existing but empty snippet directory, so the only resolvconf snippet
+// will be Tailscale's own.
+func newOpenresolvEnv(t *testing.T) (*vmtest.Env, *vmtest.Node) {
+	t.Helper()
+	env := vmtest.New(t,
+		vmtest.ControlDNS(orMagicDNSDomain, &tailcfg.DNSConfig{
+			Proxied: true, // MagicDNS, so there's a route and quad-100 is in play
+			Domains: []string{orLocalDomain},
+			Routes: map[string][]*dnstype.Resolver{
+				orLocalDomain: nil, // answer locally, from ExtraRecords
+			},
+			ExtraRecords: []tailcfg.DNSRecord{
+				{Name: orLocalName, Type: "A", Value: orLocalIP},
+			},
+		}))
+	node := env.AddNode("node",
+		env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT),
+		vmtest.OS(vmtest.Ubuntu2404),
+		vmtest.WithDNSMode(vmtest.DNSOpenresolv))
+	env.Start()
+
+	// Otherwise a pass could come from a different DNS manager.
+	env.AssertDNSBackend(node, "openresolv")
+	return env, node
+}
+
+// TestOpenresolvDNS checks that tailscaled configures DNS on a host where
+// Tailscale owns the only resolvconf snippet. Before the fix for
+// tailscale/tailscale#20825, such a host got no DNS configuration at all.
+func TestOpenresolvDNS(t *testing.T) {
+	env, node := newOpenresolvEnv(t)
+
+	// tailscaled must have taken over resolv.conf. Checking for quad-100 rather
+	// than the signature line alone matters, because natlab provisions a
+	// resolv.conf carrying that same signature.
+	assertOpenresolvResolvConf(t, env, node,
+		[]string{orSignature, orQuad100},
+		[]string{vnet.FakeDNSIPv4().String()})
+
+	assertNoDNSReadWarning(t, env, node)
+
+	// Tailscale's snippet must be registered, and it must be the only one.
+	// Otherwise the base-config check below would be running against a host
+	// this test did not set up.
+	if out, err := env.SSHExec(node, "resolvconf -i"); err != nil {
+		t.Errorf("resolvconf -i: %v (%s)", err, strings.TrimSpace(out))
+	} else if got := strings.Fields(out); !slices.Equal(got, []string{"tailscale"}) {
+		t.Errorf("resolvconf -i = %q, want just \"tailscale\"", strings.TrimSpace(out))
+	}
+
+	// Resolving through the OS resolver proves that libc really uses that
+	// resolv.conf.
+	assertResolves(t, env, node, orLocalName, orLocalIP)
+
+	// Tailscale's snippet is the only one registered, so openresolv has no OS
+	// config to report: both the nameservers and the search domains must come
+	// back empty. net/dns filters the Tailscale service IPs out of the
+	// nameservers anyway (tailscale/tailscale#7816), so the search domains are
+	// what this check really rests on.
+	base := openresolvBaseConfig(t, env, node)
+	if base == nil {
+		return // openresolvBaseConfig already reported it
+	}
+	if len(base.Nameservers) != 0 || len(base.SearchDomains) != 0 {
+		t.Errorf("OS base config = %+v, want it empty: Tailscale owns the only "+
+			"resolvconf snippet, so openresolv has no OS config to report "+
+			"(tailscale/tailscale#20825)", *base)
+	}
+}
+
+// assertOpenresolvResolvConf waits for the guest's /etc/resolv.conf to contain
+// every string in want and none in notWant.
+func assertOpenresolvResolvConf(t *testing.T, env *vmtest.Env, n *vmtest.Node, want, notWant []string) {
+	t.Helper()
+	const cmd = "cat /etc/resolv.conf"
+	var last string
+	if err := tstest.WaitFor(60*time.Second, func() error {
+		out, err := env.SSHExec(n, cmd)
+		last = out
+		if err != nil {
+			return fmt.Errorf("%s: %v (%s)", cmd, err, strings.TrimSpace(out))
+		}
+		for _, w := range want {
+			if !strings.Contains(out, w) {
+				return fmt.Errorf("resolv.conf is missing %q", w)
+			}
+		}
+		for _, w := range notWant {
+			if strings.Contains(out, w) {
+				return fmt.Errorf("resolv.conf unexpectedly has %q", w)
+			}
+		}
+		return nil
+	}); err != nil {
+		// The health lines are the fastest way to tell "DNS was never applied"
+		// from "DNS was applied but looks wrong".
+		t.Fatalf("%v\nwant all of %q and none of %q in resolv.conf:\n%s\nhealth: %q",
+			err, want, notWant, last, env.Status(n).Health)
+	}
+}
+
+// openresolvBaseConfig returns what tailscaled reads back as the node's own OS
+// DNS configuration. On failure it calls t.Errorf and returns nil.
+//
+// "tailscale dns status" fetches it over the local API, which calls straight
+// through to openresolvManager.GetBaseConfig, so this is the live result of
+// running resolvconf in the guest, not something inferred from its consequences.
+func openresolvBaseConfig(t *testing.T, env *vmtest.Env, n *vmtest.Node) *tsdnsjsonv0.SystemConfig {
+	t.Helper()
+	const cmd = "tailscale dns status --json"
+	out, err := env.SSHExec(n, cmd)
+	if err != nil {
+		t.Errorf("%s: %v (%s)", cmd, err, strings.TrimSpace(out))
+		return nil
+	}
+	var st tsdnsjsonv0.StatusResponse
+	if err := json.Unmarshal([]byte(out), &st); err != nil {
+		t.Errorf("parsing %s output: %v\n%s", cmd, err, out)
+		return nil
+	}
+	if st.SystemDNSError != "" {
+		t.Errorf("tailscaled could not read the OS DNS config: %q", st.SystemDNSError)
+		return nil
+	}
+	if st.SystemDNS == nil {
+		t.Errorf("%s reported no OS DNS config:\n%s", cmd, out)
+		return nil
+	}
+	return st.SystemDNS
+}
+
+// assertNoDNSReadWarning fails if the node is reporting that it could not read
+// the OS DNS configuration.
+func assertNoDNSReadWarning(t *testing.T, env *vmtest.Env, n *vmtest.Node) {
+	t.Helper()
+	// The warning ends with the error text, so match on what comes before it.
+	// Taking that from the Warnable rather than copying its wording matters
+	// here: this check asserts an absence, so a copy that stopped matching
+	// would leave the check passing while testing nothing.
+	want := dns.OSConfigurationReadWarnable.Text(health.Args{health.ArgError: ""})
+	for _, h := range env.Status(n).Health {
+		if strings.HasPrefix(h, want) {
+			t.Errorf("node is unhealthy: %q", h)
+		}
+	}
+}
+
+// assertResolves waits for name to resolve to want on the node. It asks for A
+// records only, so a name with both an A and a AAAA record cannot come back as
+// the wrong address family.
+func assertResolves(t *testing.T, env *vmtest.Env, n *vmtest.Node, name, want string) {
+	t.Helper()
+	cmd := "getent ahostsv4 " + name
+	if err := tstest.WaitFor(30*time.Second, func() error {
+		out, err := env.SSHExec(n, cmd)
+		if err != nil {
+			return fmt.Errorf("%s: %v (%s)", cmd, err, strings.TrimSpace(out))
+		}
+		if !strings.Contains(out, want) {
+			return fmt.Errorf("%s = %q, want it to contain %s", cmd, strings.TrimSpace(out), want)
+		}
+		return nil
+	}); err != nil {
+		out, _ := env.SSHExec(n, "cat /etc/resolv.conf; resolvconf -i")
+		t.Errorf("%v\nresolver state:\n%s", err, out)
+	}
+}

tstest/natlab/vmtest/openresolv.go

@@ -0,0 +1,168 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"embed"
+	"encoding/hex"
+	"fmt"
+	"regexp"
+)
+
+// These paths are where openresolv gets installed in the guest. openresolv's
+// build bakes them into its scripts with sed substitutions (see its Makefile),
+// so they have to agree with openresolvSubst below.
+const (
+	// openresolvSbinDir must be on the default PATH, because net/dns finds
+	// resolvconf with exec.LookPath (see resolvconfStyle in
+	// net/dns/resolvconf.go).
+	openresolvSbinDir = "/sbin"
+
+	// openresolvSysconfDir is where resolvconf looks for resolvconf.conf.
+	openresolvSysconfDir = "/etc"
+
+	// openresolvLibexecDir holds the subscriber scripts resolvconf runs after
+	// a snippet is added or removed.
+	openresolvLibexecDir = "/usr/libexec/resolvconf"
+
+	// openresolvVarDir is resolvconf's runtime state directory. Its "keys"
+	// subdirectory is where registered config snippets live.
+	openresolvVarDir = "/run/resolvconf"
+
+	// openresolvKeyDir is created empty by provisioning; see [DNSOpenresolv].
+	openresolvKeyDir = openresolvVarDir + "/keys"
+)
+
+// openresolvVersion is the upstream release the vendored sources come from.
+// The files keep their upstream names, so this is where the version is
+// recorded. See testdata/openresolv/README.md.
+const openresolvVersion = "v3.17.4"
+
+// openresolvSrc holds the vendored openresolv sources. See
+// testdata/openresolv/README.md for their provenance.
+//
+//go:embed testdata/openresolv/*.in
+var openresolvSrc embed.FS
+
+// openresolvFile is one file natlab installs into the guest to provide
+// openresolv.
+type openresolvFile struct {
+	guestPath string // absolute path to install to in the guest
+	srcName   string // vendored source to substitute, relative to testdata/openresolv
+	srcSHA256 string // hex SHA256 the vendored source must have, before substitution
+	inline    string // literal contents, for files with no upstream source
+	mode      string // octal permissions for cloud-init to apply
+}
+
+// openresolvInstall is the subset of openresolv's own files this mode installs.
+//
+// Upstream ships further subscriber scripts, for dnsmasq and unbound among
+// others. Leaving those out is safe, because each of openresolv's subscriber
+// loops skips a script that is not present. libc is the one script that has to
+// be installed, because it writes resolv.conf.
+var openresolvInstall = []openresolvFile{
+	{
+		guestPath: openresolvSbinDir + "/resolvconf",
+		srcName:   "resolvconf.in",
+		srcSHA256: "c806bd4aa0d1c59736beae3af8a7e4c7cfd6a24664cfbc01be10b28af89f5b8c",
+		mode:      "0755",
+	},
+	{
+		// resolvconf sources this rather than exec'ing it if it isn't
+		// executable, so it needs no exec bit.
+		guestPath: openresolvLibexecDir + "/libc",
+		srcName:   "libc.in",
+		srcSHA256: "25b7ba247cb033130035a09751d78be40786ee449a93f31c61b93409dabd54ea",
+		mode:      "0644",
+	},
+	{
+		// Upstream's own default, which its "make install" installs too.
+		// What matters here is that the file exists at all: without it,
+		// resolvconf switches to the original Debian layout if
+		// /etc/resolvconf happens to be a directory, which is not what this
+		// mode tests.
+		guestPath: openresolvSysconfDir + "/resolvconf.conf",
+		inline:    "resolv_conf=/etc/resolv.conf\n",
+		mode:      "0644",
+	},
+}
+
+// openresolvSubst is openresolv's build-time substitution table. Its Makefile
+// seds these placeholders out of the .in files; natlab does the same, so the
+// test needs no configure-and-make step.
+//
+// RCDIR, RESTARTCMD and STATUSARG are empty, as they are in an unconfigured
+// upstream build. resolvconf's detect_init() then picks a restart command at
+// runtime, which is systemctl on these guests. That command restarts the libc
+// service only when it is already active. Here the libc service is nscd, which
+// these guests do not run.
+var openresolvSubst = map[string]string{
+	"@SBINDIR@":    openresolvSbinDir,
+	"@SYSCONFDIR@": openresolvSysconfDir,
+	"@LIBEXECDIR@": openresolvLibexecDir,
+	"@VARDIR@":     openresolvVarDir,
+	"@RCDIR@":      "",
+	"@RESTARTCMD@": "",
+	"@STATUSARG@":  "",
+}
+
+var openresolvPlaceholderRx = regexp.MustCompile(`@[A-Z_]+@`)
+
+// substOpenresolv applies openresolvSubst to b, and fails if any placeholder
+// survives. An openresolv release that adds one would otherwise install a
+// shell script containing a literal "@FOO@", which fails in the guest in some
+// far less obvious way.
+func substOpenresolv(name string, b []byte) ([]byte, error) {
+	for k, v := range openresolvSubst {
+		b = bytes.ReplaceAll(b, []byte(k), []byte(v))
+	}
+	if m := openresolvPlaceholderRx.Find(b); m != nil {
+		return nil, fmt.Errorf("%s: unsubstituted placeholder %q; openresolvSubst needs updating for the vendored openresolv", name, m)
+	}
+	return b, nil
+}
+
+// checkOpenresolvSHA256 reports whether b is the vendored source natlab
+// expects. It does not authenticate the file against upstream, because nothing
+// here reaches the network. What it catches is a vendored file whose contents
+// changed while its recorded provenance stayed put, which is what a half-done
+// version bump or a stray local edit looks like.
+func checkOpenresolvSHA256(name string, b []byte, want string) error {
+	sum := sha256.Sum256(b)
+	if got := hex.EncodeToString(sum[:]); got != want {
+		return fmt.Errorf("testdata/openresolv/%s has SHA256 %s, want %s for openresolv %s; change the file and its srcSHA256 together, per testdata/openresolv/README.md", name, got, want, openresolvVersion)
+	}
+	return nil
+}
+
+// buildOpenresolv returns the files to install in the guest, in the order
+// openresolvInstall lists them.
+func buildOpenresolv() ([]cloudInitFile, error) {
+	out := make([]cloudInitFile, 0, len(openresolvInstall))
+	for _, f := range openresolvInstall {
+		content := []byte(f.inline)
+		if f.srcName != "" {
+			b, err := openresolvSrc.ReadFile("testdata/openresolv/" + f.srcName)
+			if err != nil {
+				return nil, err
+			}
+			// Before substitution, so the hash is the upstream file's own.
+			if err := checkOpenresolvSHA256(f.srcName, b, f.srcSHA256); err != nil {
+				return nil, err
+			}
+			if b, err = substOpenresolv(f.srcName, b); err != nil {
+				return nil, err
+			}
+			content = b
+		}
+		out = append(out, cloudInitFile{
+			path:    f.guestPath,
+			content: content,
+			mode:    f.mode,
+		})
+	}
+	return out, nil
+}

tstest/natlab/vmtest/openresolv_test.go

@@ -0,0 +1,129 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"strings"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+// TestBuildOpenresolv checks that the vendored openresolv sources still build
+// into what the guest expects. buildOpenresolv verifies their recorded hashes
+// and rejects a surviving build placeholder, so an edited source or an upstream
+// bump fails here in milliseconds. The alternative is a puzzling DNS failure
+// several minutes into TestOpenresolvDNS.
+func TestBuildOpenresolv(t *testing.T) {
+	files, err := buildOpenresolv()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != len(openresolvInstall) {
+		t.Fatalf("built %d files, want %d", len(files), len(openresolvInstall))
+	}
+
+	byPath := map[string]cloudInitFile{}
+	for _, f := range files {
+		byPath[f.path] = f
+	}
+	for _, want := range openresolvInstall {
+		f, ok := byPath[want.guestPath]
+		if !ok {
+			t.Errorf("%s: not built", want.guestPath)
+			continue
+		}
+		if len(f.content) == 0 {
+			t.Errorf("%s: empty", want.guestPath)
+		}
+		if f.mode != want.mode {
+			t.Errorf("%s: mode %q, want %q", want.guestPath, f.mode, want.mode)
+		}
+		if want.srcName != "" && !strings.HasPrefix(string(f.content), "#!/bin/sh") {
+			t.Errorf("%s: does not start with a /bin/sh shebang", want.guestPath)
+		}
+	}
+
+	// The substitutions have to have actually happened: resolvconf derives its
+	// key directory from VARDIR, and net/dns's whole openresolv path hinges on
+	// that directory being the one the test provisions.
+	rc := string(byPath[openresolvSbinDir+"/resolvconf"].content)
+	for _, want := range []string{
+		"VARDIR=" + openresolvVarDir,
+		"LIBEXECDIR=" + openresolvLibexecDir,
+		`KEYDIR="$VARDIR/keys"`,
+	} {
+		if !strings.Contains(rc, want) {
+			t.Errorf("built resolvconf does not contain %q", want)
+		}
+	}
+}
+
+// TestOpenresolvSHA256 checks the hash check in both directions. On its own,
+// TestBuildOpenresolv only ever runs it against sources that match, so a check
+// that accepted anything would still look fine there.
+func TestOpenresolvSHA256(t *testing.T) {
+	const content = "not the vendored file"
+	sum := sha256.Sum256([]byte(content))
+
+	if err := checkOpenresolvSHA256("libc.in", []byte(content), hex.EncodeToString(sum[:])); err != nil {
+		t.Errorf("matching content was rejected: %v", err)
+	}
+	if err := checkOpenresolvSHA256("libc.in", []byte(content), strings.Repeat("0", 64)); err == nil {
+		t.Error("content that does not match its recorded hash was accepted")
+	}
+}
+
+// TestOpenresolvWriteFiles checks that the guest files survive a round trip
+// through the cloud-init write_files section. resolvconf.in is 31 kB, so its
+// base64 encoding is one very long YAML scalar, and a quoting or line-length
+// mistake would show up as a guest that boots with a truncated shell script.
+func TestOpenresolvWriteFiles(t *testing.T) {
+	files, err := buildOpenresolv()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var ud strings.Builder
+	ud.WriteString("#cloud-config\n")
+	writeCloudInitFiles(&ud, files)
+
+	var got struct {
+		WriteFiles []struct {
+			Path        string `yaml:"path"`
+			Encoding    string `yaml:"encoding"`
+			Permissions string `yaml:"permissions"`
+			Content     string `yaml:"content"`
+		} `yaml:"write_files"`
+	}
+	if err := yaml.Unmarshal([]byte(ud.String()), &got); err != nil {
+		t.Fatalf("user-data is not valid YAML: %v", err)
+	}
+	if len(got.WriteFiles) != len(files) {
+		t.Fatalf("write_files has %d entries, want %d", len(got.WriteFiles), len(files))
+	}
+	for i, want := range files {
+		g := got.WriteFiles[i]
+		if g.Path != want.path {
+			t.Errorf("entry %d: path %q, want %q", i, g.Path, want.path)
+		}
+		if g.Encoding != "b64" {
+			t.Errorf("%s: encoding %q, want b64", want.path, g.Encoding)
+		}
+		if g.Permissions != want.mode {
+			t.Errorf("%s: permissions %q, want %q", want.path, g.Permissions, want.mode)
+		}
+		content, err := base64.StdEncoding.DecodeString(g.Content)
+		if err != nil {
+			t.Errorf("%s: decoding content: %v", want.path, err)
+			continue
+		}
+		if string(content) != string(want.content) {
+			t.Errorf("%s: content survived the round trip as %d bytes, want %d",
+				want.path, len(content), len(want.content))
+		}
+	}
+}

tstest/natlab/vmtest/testdata/openresolv/README.md

@@ -0,0 +1,50 @@
+# Vendored openresolv sources
+
+These are two unmodified source files from upstream openresolv, the resolvconf
+implementation behind `net/dns`'s "openresolv" backend. `DNSOpenresolv` installs
+them into a guest so a vmtest can exercise that backend. See `../../openresolv.go`.
+
+No cloud image ships openresolv, and a guest cannot download it, because vnet
+has no route to the real internet. They are vendored rather than fetched at test
+time so that running a vmtest locally needs no network access beyond the cloud
+image it already downloads.
+
+openresolv's build applies a handful of `sed` substitutions to these `.in` files
+and compiles nothing, so `openresolv.go` can do the same substitutions in
+process. That is why only the `.in` sources are here.
+
+## Provenance
+
+Both files come from commit
+[`6489889ce5631364ad2f17d391e1a3ad969619f2`](https://github.com/NetworkConfiguration/openresolv/tree/6489889ce5631364ad2f17d391e1a3ad969619f2),
+which is tagged `v3.17.4`. They are byte-identical to the same-named members of
+the `v3.17.4` release tarball.
+
+The SHA256 of each file is recorded in `openresolvInstall`, in
+`../../openresolv.go`, next to the version. `buildOpenresolv` checks both files
+against those hashes every time it runs, so a file that changes without its
+recorded hash changing too fails the tests rather than reaching a guest. The
+hashes say nothing about whether these files came from upstream, since none of
+this reaches the network. They are what makes the version above a claim the
+tests can hold the files to.
+
+To update, copy the files from a new upstream revision:
+
+```sh
+ref=<new commit>
+for f in resolvconf.in libc.in; do
+  curl -fsSLo "$f" "https://raw.githubusercontent.com/NetworkConfiguration/openresolv/$ref/$f"
+done
+sha256sum resolvconf.in libc.in
+```
+
+Then replace the commit above, and in `../../openresolv.go` replace
+`openresolvVersion` and both `srcSHA256` values. Run
+`go test ./tstest/natlab/vmtest -run 'TestBuildOpenresolv|TestOpenresolvSHA256'`
+afterwards. It fails if a hash was missed, or if the new revision adds a build
+placeholder that `openresolvSubst` does not know about.
+
+## Licensing
+
+openresolv is BSD-2-Clause, copyright Roy Marples. Each file carries the full
+license text in its header, and neither file is modified here.

tstest/natlab/vmtest/testdata/openresolv/libc.in

@@ -0,0 +1,284 @@
+#!/bin/sh
+# Copyright (c) 2007-2025 Roy Marples
+# All rights reserved
+
+# libc subscriber for resolvconf
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+SYSCONFDIR=@SYSCONFDIR@
+LIBEXECDIR=@LIBEXECDIR@
+VARDIR=@VARDIR@
+KEYDIR="$VARDIR/keys"
+# Compat
+if [ ! -d "$KEYDIR" ] && [ -d "$VARDIR/interfaces" ]; then
+	KEYDIR="$VARDIR/interfaces"
+fi
+
+CMD="$1"
+KEY="$2"
+
+NL="
+"
+
+warn()
+{
+	echo "${0##*/}: $*" >&2
+}
+
+# sed may not be available, and this is faster on small files
+key_get_value()
+{
+	key="$1"
+	shift
+
+	if [ $# -eq 0 ]; then
+		while read -r line; do
+			case "$line" in
+			"$key"*) echo "${line##$key}";;
+			esac
+		done
+	else
+		for x do
+			while read -r line; do
+				case "$line" in
+				"$key"*) echo "${line##$key}";;
+				esac
+			done < "$x"
+		done
+	fi
+}
+
+keys_remove()
+{
+	while read -r line; do
+		found=false
+		for key do
+			case "$line" in
+			"$key"*|"#"*|" "*|"	"*|"") found=true;;
+			esac
+			$found && break
+		done
+		$found || echo "$line"
+	done
+}
+
+local_nameservers="127.* 0.0.0.0 255.255.255.255 ::1"
+
+# Support original resolvconf configuration layout
+# as well as the openresolv config file
+if [ -f "$SYSCONFDIR"/resolvconf.conf ]; then
+	. "$SYSCONFDIR"/resolvconf.conf
+elif [ -d "$SYSCONFDIR"/resolvconf ]; then
+	SYSCONFDIR="$SYSCONFDIR/resolvconf"
+	base="$SYSCONFDIR/resolv.conf.d/base"
+	if [ -f "$base" ]; then
+		prepend_nameservers="$(key_get_value "nameserver " "$base")"
+		domain="$(key_get_value "domain " "$base")"
+		prepend_search="$(key_get_value "search " "$base")"
+		resolv_conf_options="$(key_get_value "options " "$base")"
+		resolv_conf_sortlist="$(key_get_value "sortlist " "$base")"
+	fi
+	if [ -f "$SYSCONFDIR"/resolv.conf.d/head ]; then
+		resolv_conf_head="$(cat "${SYSCONFDIR}"/resolv.conf.d/head)"
+	fi
+	if [ -f "$SYSCONFDIR"/resolv.conf.d/tail ]; then
+		resolv_conf_tail="$(cat "$SYSCONFDIR"/resolv.conf.d/tail)"
+	fi
+fi
+: ${resolv_conf:=/etc/resolv.conf}
+if [ "$resolv_conf" = "/dev/null" ]; then
+	exit 0
+fi
+: ${resolv_conf_tmp:="$resolv_conf.$$.openresolv"}
+: ${libc_service:=nscd}
+: ${list_resolv:=@SBINDIR@/resolvconf -L}
+if [ "${resolv_conf_head-x}" = x ] && [ -f "$SYSCONFDIR"/resolv.conf.head ]
+then
+	resolv_conf_head="$(cat "${SYSCONFDIR}"/resolv.conf.head)"
+fi
+if [ "${resolv_conf_tail-x}" = x ] && [ -f "$SYSCONFDIR"/resolv.conf.tail ]
+then
+	resolv_conf_tail="$(cat "$SYSCONFDIR"/resolv.conf.tail)"
+fi
+
+signature="# Generated by resolvconf"
+
+uniqify()
+{
+	result=
+	while [ -n "$1" ]; do
+		case " $result " in
+		*" $1 "*);;
+		*) result="$result $1";;
+		esac
+		shift
+	done
+	echo "${result# *}"
+}
+
+case "${resolv_conf_passthrough:-NO}" in
+[Yy][Ee][Ss]|[Tt][Rr][Uu][Ee]|[Oo][Nn]|1)
+	newest=
+	for conf in "$KEYDIR"/*; do
+		if [ -z "$newest" ] || [ "$conf" -nt "$newest" ]; then
+			newest="$conf"
+		fi
+	done
+	[ -z "$newest" ] && exit 0
+	newconf="$signature$NL$(cat "$newest")$NL"
+	;;
+/dev/null|[Nn][Uu][Ll][Ll])
+	: ${resolv_conf_local_only:=NO}
+	if [ "$local_nameservers" = "127.* 0.0.0.0 255.255.255.255 ::1" ]; then
+		local_nameservers=
+	fi
+	# Need to overwrite our variables.
+	eval "$(@SBINDIR@/resolvconf -V)"
+	;;
+
+*)
+	[ -z "$RESOLVCONF" ] && eval "$(@SBINDIR@/resolvconf -v)"
+	;;
+esac
+case "${resolv_conf_passthrough:-NO}" in
+[Yy][Ee][Ss]|[Tt][Rr][Uu][Ee]|[Oo][Nn]|1) ;;
+*)
+	: ${domain:=$DOMAIN}
+	newsearch="$(uniqify $prepend_search $SEARCH $append_search)"
+	NS="$LOCALNAMESERVERS $NAMESERVERS"
+	newns=
+	gotlocal=false
+	for n in $(uniqify $prepend_nameservers $NS $append_nameservers); do
+		add=true
+		islocal=false
+		for l in $local_nameservers; do
+			case "$n" in
+			$l) islocal=true; gotlocal=true; break;;
+			esac
+		done
+		if ! $islocal; then
+			case "${resolv_conf_local_only:-YES}" in
+			[Yy][Ee][Ss]|[Tt][Rr][Uu][Ee]|[Oo][Nn]|1)
+				$gotlocal && add=false;;
+			esac
+		fi
+		$add && newns="$newns $n"
+	done
+
+	# Hold our new resolv.conf in a variable to save on temporary files
+	newconf="$signature$NL"
+	if [ -n "$resolv_conf_head" ]; then
+		newconf="$newconf$resolv_conf_head$NL"
+	fi
+
+	[ -n "$domain" ] && newconf="${newconf}domain $domain$NL"
+	if [ -n "$newsearch" ] && [ "$newsearch" != "$domain" ]; then
+		newconf="${newconf}search $newsearch$NL"
+	fi
+	for n in $newns; do
+		newconf="${newconf}nameserver $n$NL"
+	done
+
+	# Now add anything we don't care about such as sortlist and options
+	stuff="$($list_resolv | keys_remove nameserver domain search)"
+	if [ -n "$stuff" ]; then
+		newconf="$newconf$stuff$NL"
+	fi
+
+	# Append any user defined ones
+	if [ -n "$resolv_conf_options" ]; then
+		newconf="${newconf}options $resolv_conf_options$NL"
+	fi
+	if [ -n "$resolv_conf_sortlist" ]; then
+		newconf="${newconf}sortlist $resolv_conf_sortlist$NL"
+	fi
+
+	if [ -n "$resolv_conf_tail" ]; then
+		newconf="$newconf$resolv_conf_tail$NL"
+	fi
+	;;
+esac
+
+# Check if the file has actually changed or not
+if [ -e "$resolv_conf" ]; then
+	if [ "$CMD" != u ] && \
+	    [ "$(cat "$resolv_conf")" = "$(printf %s "$newconf")" ]
+	then
+		exit 0
+	fi
+	read line <"$resolv_conf"
+	if [ "$line" != "$signature" ]; then
+		if [ "$CMD" != u ]; then
+			warn "signature mismatch: $resolv_conf"
+			warn "run \`resolvconf -u\` to update"
+			exit 1
+		fi
+		cp "$resolv_conf" "$resolv_conf.bak"
+	fi
+fi
+
+# There are pros and cons for writing directly to resolv.conf
+# instead of a temporary file and then moving it over.
+# The default is to write to resolv.conf as it has the least
+# issues and has been the long standing default behaviour.
+# resolv.conf could also be bind mounted for network namespaces
+# so we cannot move in this instance.
+case "${resolv_conf_mv:-NO}" in
+[Yy][Ee][Ss]|[Tt][Rr][Uu][Ee]|[Oo][Nn]|1)
+	# Protect against symlink attack, ensure new file does not exist
+	rm -f "$resolv_conf_tmp"
+	# Keep original file owner, group and mode
+	[ -r "$resolv_conf" ] && cp -p "$resolv_conf" "$resolv_conf_tmp"
+	# Create our resolv.conf now
+	if (umask 022; printf %s "$newconf" >"$resolv_conf_tmp"); then
+		mv "$resolv_conf_tmp" "$resolv_conf"
+	fi
+	;;
+*)
+	(umask 022; printf %s "$newconf" >"$resolv_conf")
+	;;
+esac
+
+if [ -n "$libc_restart" ]; then
+	eval $libc_restart
+elif [ -n "$RESTARTCMD" ]; then
+	set -- ${libc_service}
+	eval "$RESTARTCMD"
+else
+	@SBINDIR@/resolvconf -r ${libc_service}
+fi
+
+retval=0
+# Notify users of the resolver
+for script in "$LIBEXECDIR"/libc.d/*; do
+	if [ -f "$script" ]; then
+		if [ -x "$script" ]; then
+			"$script" "$@"
+		else
+			(. "$script")
+		fi
+		retval=$(($retval + $?))
+	fi
+done
+exit $retval

tstest/natlab/vmtest/vmtest.go

@@ -489,7 +489,7 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 			n.webServerPort = int(o)
 		case nodeOptDNSMode:
 			switch DNSMode(o) {
-			case DNSDefault, DNSDirect:
+			case DNSDefault, DNSDirect, DNSOpenresolv:
 			default:
 				e.t.Fatalf("AddNode(%q): unsupported DNSMode %q", name, DNSMode(o))
 			}
@@ -579,6 +579,16 @@ type nodeOptSystemdUnit struct{
 	// DNSDirect masks systemd-resolved and installs a plain /etc/resolv.conf
 	// so tailscaled selects the "direct" manager (rewrites resolv.conf itself).
 	DNSDirect DNSMode = "direct"
+
+	// DNSOpenresolv masks systemd-resolved and installs upstream openresolv
+	// (which no cloud image ships), so tailscaled selects the "openresolv"
+	// manager. Its key directory is created empty, so the only snippet ever
+	// registered is Tailscale's own. Before the fix for tailscale/tailscale#20825,
+	// tailscaled handled that state wrong. openresolv reports "no snippets" by
+	// exiting 2, and net/dns treated that non-zero exit as a hard failure.
+	//
+	// openresolv's sources are vendored into the tree; see openresolv.go.
+	DNSOpenresolv DNSMode = "openresolv"
 )
 
 // OS returns a NodeOption that sets the node's operating system image.