cmd/k8s-operator/e2e: add tests to cover DNSConfig (#20861)

Deploy k8s nameserver during e2e test setup, and point the cluster resolver
(CoreDNS or kube-dns) to it so tests can resolve MagicDNS names inside the
cluster.

Add a test to verify singleton L7 Ingress is reachable from inside the
cluster using its MagicDNS Name.

Update existing egress tests to use a dedicated tailnet target per test (to
avoid conflicts). Egress tests now also verify that an egress target is
reachable from within the cluster using both its Service and MagicDNS name.

To successfully curl using the target's MagicDNS name, publish test CAs as a
ConfigMap to the cluster, and mount these for each curl pod.

Fixes tailscale/corp#38027

Signed-off-by: Becky Pauley <becky@tailscale.com>

cmd/k8s-operator/e2e/egress_test.go

@@ -4,13 +4,19 @@
 package e2e
 
 import (
+	"errors"
 	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"strings"
 	"testing"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"tailscale.com/client/tailscale/v2"
 	kube "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/tstest"
@@ -18,6 +24,12 @@
 
 const egressPort = 80
 
+// tailnetTarget holds the FQDN, IPv4, and IPv6 addresses of a tailnet node
+// running an HTTP server, for use as an egress target in tests.
+type tailnetTarget struct {
+	fqdn, ipv4, ipv6 string
+}
+
 // See [TestMain] for test requirements.
 func TestEgress(t *testing.T) {
 	if tnClient == nil {
@@ -25,16 +37,18 @@ func TestEgress(t *testing.T) {
 	}
 	t.Parallel()
 
+	target := startEgressTarget(t, tsClient)
+
 	t.Run("IPv4", func(t *testing.T) {
 		if !clusterIPv4Support {
 			t.Skip("cluster does not support IPv4")
 		}
 		svc := egressService(generateName("test-egress"), map[string]string{
-			"tailscale.com/tailnet-ip": tnTarget.ipv4,
+			"tailscale.com/tailnet-ip": target.ipv4,
 		})
 		createAndCleanup(t, kubeClient, svc)
 		waitForEgress(t, svc.Name, kube.SvcIsReady)
-		testEgressIsReachable(t, ns, svc.Name)
+		requireTargetIsReachable(t, egressURL(svc.Name))
 	})
 
 	t.Run("IPv6", func(t *testing.T) {
@@ -42,20 +56,20 @@ func TestEgress(t *testing.T) {
 			t.Skip("cluster does not support IPv6")
 		}
 		svc := egressService(generateName("test-egress"), map[string]string{
-			"tailscale.com/tailnet-ip": tnTarget.ipv6,
+			"tailscale.com/tailnet-ip": target.ipv6,
 		})
 		createAndCleanup(t, kubeClient, svc)
 		waitForEgress(t, svc.Name, kube.SvcIsReady)
-		testEgressIsReachable(t, ns, svc.Name)
+		requireTargetIsReachable(t, egressURL(svc.Name))
 	})
 
 	t.Run("FQDN", func(t *testing.T) {
 		svc := egressService(generateName("test-egress"), map[string]string{
-			"tailscale.com/tailnet-fqdn": tnTarget.fqdn,
+			"tailscale.com/tailnet-fqdn": target.fqdn,
 		})
 		createAndCleanup(t, kubeClient, svc)
 		waitForEgress(t, svc.Name, kube.SvcIsReady)
-		testEgressIsReachable(t, ns, svc.Name)
+		requireTargetIsReachable(t, egressURL(svc.Name))
 	})
 }
 
@@ -66,6 +80,8 @@ func TestHAEgress(t *testing.T) {
 	}
 	t.Parallel()
 
+	target := startEgressTarget(t, tsClient)
+
 	pg := &tsapi.ProxyGroup{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: generateName("egress"),
@@ -81,12 +97,12 @@ func TestHAEgress(t *testing.T) {
 			t.Skip("cluster does not support IPv4")
 		}
 		svc := egressService(generateName("test-egress"), map[string]string{
-			"tailscale.com/tailnet-ip":  tnTarget.ipv4,
+			"tailscale.com/tailnet-ip":  target.ipv4,
 			"tailscale.com/proxy-group": pg.Name,
 		})
 		createAndCleanup(t, kubeClient, svc)
 		waitForEgress(t, svc.Name, pgEgressReady)
-		testEgressIsReachable(t, ns, svc.Name)
+		requireTargetIsReachable(t, egressURL(svc.Name))
 	})
 
 	t.Run("IPv6", func(t *testing.T) {
@@ -94,22 +110,24 @@ func TestHAEgress(t *testing.T) {
 			t.Skip("cluster does not support IPv6")
 		}
 		svc := egressService(generateName("test-egress"), map[string]string{
-			"tailscale.com/tailnet-ip":  tnTarget.ipv6,
+			"tailscale.com/tailnet-ip":  target.ipv6,
 			"tailscale.com/proxy-group": pg.Name,
 		})
 		createAndCleanup(t, kubeClient, svc)
 		waitForEgress(t, svc.Name, pgEgressReady)
-		testEgressIsReachable(t, ns, svc.Name)
+		requireTargetIsReachable(t, egressURL(svc.Name))
 	})
 
 	t.Run("FQDN", func(t *testing.T) {
 		svc := egressService(generateName("test-egress"), map[string]string{
-			"tailscale.com/tailnet-fqdn": tnTarget.fqdn,
+			"tailscale.com/tailnet-fqdn": target.fqdn,
 			"tailscale.com/proxy-group":  pg.Name,
 		})
 		createAndCleanup(t, kubeClient, svc)
 		waitForEgress(t, svc.Name, pgEgressReady)
-		testEgressIsReachable(t, ns, svc.Name)
+		// Test the egress target is reachable via both its in-cluster Service name, and its Magic DNS name.
+		requireTargetIsReachable(t, egressURL(svc.Name))
+		requireTargetIsReachable(t, fmt.Sprintf("http://%s:%d", target.fqdn, egressPort))
 	})
 }
 
@@ -134,13 +152,51 @@ func TestHAEgressMultiTailnet(t *testing.T) {
 		t.Fatalf("verifying ProxyGroup %s is registered to the correct tailnet: %v", pg.Name, err)
 	}
 
+	target := startEgressTarget(t, secondTSClient)
 	svc := egressService(generateName("test-egress"), map[string]string{
-		"tailscale.com/tailnet-fqdn": secondTNTarget.fqdn,
+		"tailscale.com/tailnet-fqdn": target.fqdn,
 		"tailscale.com/proxy-group":  pg.Name,
 	})
 	createAndCleanup(t, kubeClient, svc)
 	waitForEgress(t, svc.Name, pgEgressReady)
-	testEgressIsReachable(t, ns, svc.Name)
+	// Test the egress target is reachable via both its in-cluster Service name, and its Magic DNS name.
+	requireTargetIsReachable(t, egressURL(svc.Name))
+	requireTargetIsReachable(t, fmt.Sprintf("http://%s:%d", target.fqdn, egressPort))
+}
+
+func startEgressTarget(t *testing.T, cl *tailscale.Client) tailnetTarget {
+	t.Helper()
+	srv := newTailnetNode(t, cl, generateName("test-egress-target"))
+
+	ln, err := srv.Listen("tcp", fmt.Sprintf(":%d", egressPort))
+	if err != nil {
+		t.Fatalf("listening on egress target: %v", err)
+	}
+	go func() {
+		if err := http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+		})); err != nil && !errors.Is(err, net.ErrClosed) {
+			log.Printf("egress target HTTP server exited: %v", err)
+		}
+	}()
+
+	lc, err := srv.LocalClient()
+	if err != nil {
+		t.Fatalf("getting egress target local client: %v", err)
+	}
+	status, err := lc.StatusWithoutPeers(t.Context())
+	if err != nil {
+		t.Fatalf("getting egress target status: %v", err)
+	}
+	target := tailnetTarget{fqdn: strings.TrimSuffix(status.Self.DNSName, ".")}
+	for _, ip := range status.TailscaleIPs {
+		if ip.Is4() {
+			target.ipv4 = ip.String()
+		} else {
+			target.ipv6 = ip.String()
+		}
+	}
+	return target
 }
 
 func egressService(name string, annotations map[string]string) *corev1.Service {
@@ -169,6 +225,10 @@ func pgEgressReady(svc *corev1.Service) bool {
 	return cond != nil && cond.Status == metav1.ConditionTrue
 }
 
+func egressURL(svcName string) string {
+	return fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", svcName, ns, egressPort)
+}
+
 func waitForEgress(t *testing.T, svcName string, ready func(*corev1.Service) bool) {
 	t.Helper()
 	if err := tstest.WaitFor(5*time.Minute, func() error {
@@ -185,43 +245,3 @@ func waitForEgress(t *testing.T, svcName string, ready func(*corev1.Service) boo
 		t.Fatalf("error waiting for Service %s to become ready: %v", svcName, err)
 	}
 }
-
-func testEgressIsReachable(t *testing.T, namespace, svcName string) {
-	t.Helper()
-	url := fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", svcName, namespace, egressPort)
-	pod := &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      generateName("curl"),
-			Namespace: namespace,
-		},
-		Spec: corev1.PodSpec{
-			RestartPolicy: corev1.RestartPolicyNever,
-			Containers: []corev1.Container{
-				{
-					Name:  "curl",
-					Image: "curlimages/curl",
-					Command: []string{"sh", "-c", fmt.Sprintf(
-						`for i in $(seq 1 10); do `+
-							`code=$(curl -s -o /dev/null -w "%%{http_code}" --max-time 5 %q); `+
-							`[ "$code" = "200" ] && exit 0; sleep 2; done; exit 1`, url)},
-				},
-			},
-		},
-	}
-	createAndCleanup(t, kubeClient, pod)
-
-	if err := tstest.WaitFor(2*time.Minute, func() error {
-		p := &corev1.Pod{ObjectMeta: objectMeta(namespace, pod.Name)}
-		if err := get(t.Context(), kubeClient, p); err != nil {
-			return err
-		}
-		if p.Status.Phase == corev1.PodSucceeded {
-			t.Logf("curl pod %s succeeded", pod.Name)
-			return nil
-		}
-		return fmt.Errorf("curl pod %s phase: %s", pod.Name, p.Status.Phase)
-	}); err != nil {
-		t.Fatalf("egress service %s/%s not reachable: %v",
-			namespace, svcName, err)
-	}
-}

cmd/k8s-operator/e2e/helpers_test.go

@@ -4,6 +4,7 @@
 package e2e
 
 import (
+	"context"
 	"crypto/rand"
 	"crypto/tls"
 	"fmt"
@@ -13,8 +14,10 @@
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
+	"tailscale.com/client/tailscale/v2"
+	"tailscale.com/ipn/store/mem"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tsnet"
@@ -25,6 +28,72 @@ func generateName(prefix string) string {
 	return fmt.Sprintf("%s-%s", prefix, strings.ToLower(rand.Text()))
 }
 
+func requireTargetIsReachable(t *testing.T, url string) {
+	t.Helper()
+
+	volumes, mounts, cacertFlag := certVolumesForURL(url)
+
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      generateName("curl"),
+			Namespace: ns,
+		},
+		Spec: corev1.PodSpec{
+			RestartPolicy: corev1.RestartPolicyNever,
+			Volumes:       volumes,
+			Containers: []corev1.Container{
+				{
+					Name:         "curl",
+					Image:        "curlimages/curl",
+					VolumeMounts: mounts,
+					Command: []string{"sh", "-c", fmt.Sprintf(
+						`for i in $(seq 1 40); do `+
+							`code=$(curl -s %s-o /dev/null -w "%%{http_code}" --max-time 5 %q); `+
+							`[ "$code" = "200" ] && exit 0; sleep 2; done; exit 1`, cacertFlag, url)},
+				},
+			},
+		},
+	}
+	createAndCleanup(t, kubeClient, pod)
+
+	if err := tstest.WaitFor(5*time.Minute, func() error {
+		p := &corev1.Pod{ObjectMeta: objectMeta(ns, pod.Name)}
+		if err := get(t.Context(), kubeClient, p); err != nil {
+			return err
+		}
+		if p.Status.Phase == corev1.PodSucceeded {
+			t.Logf("curl pod %s succeeded", pod.Name)
+			return nil
+		}
+		if p.Status.Phase == corev1.PodFailed {
+			t.Fatalf("%s not reachable in-cluster: curl pod %s failed", url, pod.Name)
+		}
+		return fmt.Errorf("curl pod %s phase: %s", pod.Name, p.Status.Phase)
+	}); err != nil {
+		t.Fatalf("%s not reachable in-cluster: %v", url, err)
+	}
+}
+
+// certVolumesForURL returns the Pod volume, VolumeMount, and curl "--cacert"
+// argument needed to verify an HTTPS url against the test CAs published in the testCAsConfigMap.
+func certVolumesForURL(url string) ([]corev1.Volume, []corev1.VolumeMount, string) {
+	if !strings.HasPrefix(url, "https://") {
+		return nil, nil, ""
+	}
+	const mountPath = "/etc/test-cas"
+	volumes := []corev1.Volume{{
+		Name: "test-cas",
+		VolumeSource: corev1.VolumeSource{
+			ConfigMap: &corev1.ConfigMapVolumeSource{
+				LocalObjectReference: corev1.LocalObjectReference{Name: testCAsConfigMap},
+			},
+		},
+	}}
+	mounts := []corev1.VolumeMount{{Name: "test-cas", MountPath: mountPath, ReadOnly: true}}
+	cacertFlag := "--cacert " + mountPath + "/" + testCAsConfigMapKey + " "
+	return volumes, mounts, cacertFlag
+}
+
 // newHTTPClient returns a HTTP client for the given tailnet client.
 // When running against devcontrol, trusts Pebble testCAs. Otherwise,
 // trusts Let's Encrypt staging testCA.
@@ -130,3 +199,29 @@ func verifyProxyGroupTailnet(t *testing.T, pg *tsapi.ProxyGroup, cl *tsnet.Serve
 	}
 	return nil
 }
+
+func newTailnetNode(t *testing.T, cl *tailscale.Client, hostname string) *tsnet.Server {
+	t.Helper()
+	caps := tailscale.KeyCapabilities{}
+	caps.Devices.Create.Preauthorized = true
+	caps.Devices.Create.Ephemeral = true
+	caps.Devices.Create.Tags = []string{"tag:k8s"}
+	authKey, err := cl.Keys().CreateAuthKey(t.Context(), tailscale.CreateKeyRequest{Capabilities: caps})
+	if err != nil {
+		t.Fatalf("creating auth key: %v", err)
+	}
+	t.Cleanup(func() { cl.Keys().Delete(context.Background(), authKey.ID) })
+
+	srv := &tsnet.Server{
+		ControlURL: cl.BaseURL.String(),
+		Hostname:   hostname,
+		Ephemeral:  true,
+		Store:      &mem.Store{},
+		AuthKey:    authKey.Key,
+	}
+	if _, err := srv.Up(t.Context()); err != nil {
+		t.Fatalf("bringing up node: %v", err)
+	}
+	t.Cleanup(func() { srv.Close() })
+	return srv
+}

cmd/k8s-operator/e2e/ingress_test.go

@@ -227,6 +227,43 @@ func TestL7Ingress(t *testing.T) {
 	}
 }
 
+func TestL7IngressDNSConfig(t *testing.T) {
+	if tnClient == nil {
+		t.Skip("TestL7IngressDNSConfig requires a working tailnet client")
+	}
+	t.Parallel()
+
+	nginx := nginxDeployment(ns)
+	createAndCleanup(t, kubeClient, nginx)
+	createAndCleanup(t, kubeClient, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      nginx.Name,
+			Namespace: ns,
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"app.kubernetes.io/name": nginx.Name,
+			},
+			Ports: []corev1.ServicePort{
+				{
+					Name: "http",
+					Port: 80,
+				},
+			},
+		},
+	})
+
+	ingress := l7Ingress(ns, nginx.Name, map[string]string{
+		"tailscale.com/experimental-forward-cluster-traffic-via-ingress": "true",
+	})
+	createAndCleanup(t, kubeClient, ingress)
+	hostname, err := waitForIngressHostname(t, ns, ingress.Name)
+	if err != nil {
+		t.Fatalf("error waiting for Ingress hostname: %v", err)
+	}
+	requireTargetIsReachable(t, fmt.Sprintf("https://%s:443", hostname))
+}
+
 func TestL7HAIngress(t *testing.T) {
 	if tnClient == nil {
 		t.Skip("TestL7HAIngress requires a working tailnet client")

cmd/k8s-operator/e2e/setup.go

@@ -11,12 +11,10 @@
 	"crypto/x509"
 	_ "embed"
 	"encoding/json"
-	"errors"
 	"flag"
 	"fmt"
 	"io"
-	"log"
-	"net"
+	"maps"
 	"net/http"
 	"net/url"
 	"os"
@@ -68,19 +66,19 @@
 )
 
 const (
-	pebbleTag       = "2.8.0"
-	ns              = "default"
-	tmp             = "/tmp/k8s-operator-e2e"
-	kindClusterName = "k8s-operator-e2e"
+	pebbleTag           = "2.8.0"
+	ns                  = "default"
+	tmp                 = "/tmp/k8s-operator-e2e"
+	kindClusterName     = "k8s-operator-e2e"
+	testCAsConfigMap    = "test-cas"
+	testCAsConfigMapKey = "test-cas.pem"
 )
 
 var (
 	tsClient           *tailscale.Client // For API calls to control.
 	tnClient           *tsnet.Server     // For testing real tailnet traffic on first tailnet.
-	tnTarget           tailnetTarget     // Egress target on the first tailnet.
 	secondTSClient     *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet).
 	secondTNClient     *tsnet.Server     // For testing real tailnet traffic on second tailnet.
-	secondTNTarget     tailnetTarget     // Egress target on the second tailnet.
 	restCfg            *rest.Config      // For constructing a client-go client if necessary.
 	kubeClient         client.WithWatch  // For k8s API calls.
 	clusterLoginServer string
@@ -187,11 +185,12 @@ func runTests(m *testing.M) (int, error) {
 	}
 
 	var (
-		clientID, clientSecret string   // OAuth client for the first tailnet (for the operator to use).
-		caPaths                []string // Extra CA cert file paths to add to images.
+		clientID, clientSecret             string // OAuth client for the first tailnet (for the operator to use).
+		secondClientID, secondClientSecret string // OAuth client for the second tailnet (for the operator to use).
 
-		certsDir                           = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images.
-		secondClientID, secondClientSecret string                        // OAuth client for the second tailnet (for the operator to use).
+		caPaths  []string                      // Extra CA cert file paths to add to images.
+		certsDir = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images.
+		caPEM    []byte                        // Used to collect and then publish test-cas ConfigMap.
 	)
 	testCAs = x509.NewCertPool()
 	if *fDevcontrol {
@@ -225,6 +224,7 @@ func runTests(m *testing.M) (int, error) {
 		if ok := testCAs.AppendCertsFromPEM(pebbleCAChain); !ok {
 			return 0, fmt.Errorf("failed to parse pebble ca chain cert")
 		}
+		caPEM = appendPEM(caPEM, pebbleMiniCACert, pebbleCAChain)
 
 		if err = os.MkdirAll(certsDir, 0755); err != nil {
 			return 0, fmt.Errorf("failed to create certs dir: %w", err)
@@ -368,7 +368,7 @@ func runTests(m *testing.M) (int, error) {
 		if ok := testCAs.AppendCertsFromPEM(leStagingRootX1); !ok {
 			return 0, fmt.Errorf("failed to parse Let's Encrypt staging root")
 		}
-
+		caPEM = appendPEM(caPEM, leStagingRootX1)
 		clientSecret = os.Getenv("TS_API_CLIENT_SECRET")
 		if clientSecret == "" {
 			return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator")
@@ -395,6 +395,19 @@ func runTests(m *testing.M) (int, error) {
 		}
 	}
 
+	// Publish the trustedCAs as a ConfigMap that can be used by in-cluster
+	// testing workloads.
+	if len(caPEM) > 0 {
+		caCM := &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{Name: testCAsConfigMap, Namespace: ns},
+			Data:       map[string]string{testCAsConfigMapKey: string(caPEM)},
+		}
+		if err := createOrUpdate(ctx, kubeClient, caCM); err != nil {
+			return 0, fmt.Errorf("failed to publish test CAs ConfigMap: %w", err)
+		}
+		defer kubeClient.Delete(context.Background(), caCM)
+	}
+
 	var ossTag string
 	if *fBuild {
 		// TODO(tomhjp): proper support for --build=false and layering pebble certs on top of existing images.
@@ -411,9 +424,10 @@ func runTests(m *testing.M) (int, error) {
 			logger.Infof("using base image: %q", *fBaseImage)
 		}
 		ossImageToTarget := map[string]string{
-			"local/k8s-operator": "publishdevoperator",
-			"local/tailscale":    "publishdevimage",
-			"local/k8s-proxy":    "publishdevproxy",
+			"local/k8s-operator":   "publishdevoperator",
+			"local/tailscale":      "publishdevimage",
+			"local/k8s-proxy":      "publishdevproxy",
+			"local/k8s-nameserver": "publishdevnameserver",
 		}
 		for img, target := range ossImageToTarget {
 			if err := buildImage(ctx, ossDir, img, target, ossTag, *fBaseImage, caPaths); err != nil {
@@ -519,6 +533,26 @@ func runTests(m *testing.M) (int, error) {
 		return 0, fmt.Errorf("failed to apply default ProxyClass: %w", err)
 	}
 
+	// Leave the nameserver image unset when nothing was built so
+	// the operator falls back to the default.
+	// TODO(beckypauley): fix for other images where build is false.
+	nameserverImg := &tsapi.NameserverImage{}
+	if ossTag != "" {
+		nameserverImg.Repo = "local/k8s-nameserver"
+		nameserverImg.Tag = ossTag
+	}
+	dnsConfig, err := deployNameserver(ctx, logger, kubeClient, nameserverImg)
+	if err != nil {
+		return 0, fmt.Errorf("failed to deploy nameserver: %w", err)
+	}
+	defer kubeClient.Delete(context.Background(), dnsConfig)
+
+	restoreClusterDNS, err := patchClusterDNS(ctx, logger, dnsConfig.Status.Nameserver.IP)
+	if err != nil {
+		return 0, fmt.Errorf("failed to patch cluster DNS: %w", err)
+	}
+	defer restoreClusterDNS()
+
 	caps := tailscale.KeyCapabilities{}
 	caps.Devices.Create.Preauthorized = true
 	caps.Devices.Create.Ephemeral = true
@@ -548,10 +582,6 @@ func runTests(m *testing.M) (int, error) {
 		return 0, err
 	}
 	defer tnClient.Close()
-	tnTarget, err = startTailnetHTTPServer(ctx, tnClient)
-	if err != nil {
-		return 0, fmt.Errorf("failed to start tailnet HTTP server on first tailnet: %w", err)
-	}
 
 	secondTNClient = &tsnet.Server{
 		ControlURL: secondTSClient.BaseURL.String(),
@@ -565,10 +595,6 @@ func runTests(m *testing.M) (int, error) {
 		return 0, err
 	}
 	defer secondTNClient.Close()
-	secondTNTarget, err = startTailnetHTTPServer(ctx, secondTNClient)
-	if err != nil {
-		return 0, fmt.Errorf("failed to start tailnet HTTP server on second tailnet: %w", err)
-	}
 
 	// Create the tailnet Secret in the tailscale namespace.
 	secret := &corev1.Secret{
@@ -743,6 +769,124 @@ func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl c
 	return nil
 }
 
+// appendPEM joins PEM blobs with a newline separator so a blob lacking a
+// trailing newline doesn't glue its END line to the next BEGIN line.
+// Otherwise curl/OpenSSL can silently drop the later certs.
+// TODO(beckypauley):  avoid maintaining both caPEM and testCAs. This can then be removed.
+func appendPEM(dst []byte, blobs ...[]byte) []byte {
+	for _, b := range blobs {
+		if len(b) == 0 {
+			continue
+		}
+		if len(dst) > 0 && dst[len(dst)-1] != '\n' {
+			dst = append(dst, '\n')
+		}
+		dst = append(dst, b...)
+	}
+	return dst
+}
+
+func deployNameserver(ctx context.Context, logger *zap.SugaredLogger, cl client.Client, img *tsapi.NameserverImage) (*tsapi.DNSConfig, error) {
+	dc := &tsapi.DNSConfig{
+		ObjectMeta: metav1.ObjectMeta{Name: "dns"},
+		Spec:       tsapi.DNSConfigSpec{Nameserver: &tsapi.Nameserver{Image: img}},
+	}
+	if err := createOrUpdate(ctx, cl, dc); err != nil {
+		return nil, fmt.Errorf("failed to create DNSConfig: %w", err)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer cancel()
+	ticker := time.NewTicker(time.Second * 2)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("timeout waiting for nameserver to be ready")
+		case <-ticker.C:
+			if err := cl.Get(ctx, client.ObjectKeyFromObject(dc), dc); err != nil {
+				return nil, fmt.Errorf("failed to get DNSConfig: %w", err)
+			}
+			if tsoperator.DNSCfgIsReady(dc) && dc.Status.Nameserver != nil && dc.Status.Nameserver.IP != "" {
+				logger.Infof("nameserver ready; Service IP %s", dc.Status.Nameserver.IP)
+				return dc, nil
+			}
+			logger.Info("waiting for nameserver to be ready...")
+		}
+	}
+}
+
+func patchClusterDNS(ctx context.Context, logger *zap.SugaredLogger, nameserverIP string) (func(), error) {
+	if cm := getDNSConfigMap(ctx, "coredns"); cm != nil && cm.Data["Corefile"] != "" {
+		corefile := stripTSNetZone(cm.Data["Corefile"]) + fmt.Sprintf(`
+ts.net:53 {
+    errors
+    cache 30
+    forward . %s
+}
+`, nameserverIP)
+		return patchDNSConfigMap(logger, cm, "Corefile", corefile)
+	}
+	if cm := getDNSConfigMap(ctx, "kube-dns"); cm != nil {
+		stub, err := json.Marshal(map[string][]string{"ts.net": {nameserverIP}})
+		if err != nil {
+			return nil, fmt.Errorf("marshalling stubDomains: %w", err)
+		}
+		return patchDNSConfigMap(logger, cm, "stubDomains", string(stub))
+	}
+	return nil, fmt.Errorf("cluster DNS is not a patchable CoreDNS/kube-dns")
+}
+
+func getDNSConfigMap(ctx context.Context, name string) *corev1.ConfigMap {
+	cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "kube-system", Name: name}}
+	if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
+		return nil
+	}
+	return cm
+}
+
+// patchDNSConfigMap updates the given Configmap and returns a closure
+// for test cleanup.
+func patchDNSConfigMap(logger *zap.SugaredLogger, cm *corev1.ConfigMap, key, value string) (func(), error) {
+	orig := maps.Clone(cm.Data)
+	if cm.Data == nil {
+		cm.Data = map[string]string{}
+	}
+	cm.Data[key] = value
+	if err := kubeClient.Update(context.Background(), cm); err != nil {
+		return nil, fmt.Errorf("patching %s %s: %w", cm.Name, key, err)
+	}
+	logger.Infof("patched %s %s with ts.net entry", cm.Name, key)
+
+	name := cm.Name
+	return func() {
+		restore := getDNSConfigMap(context.Background(), name)
+		if restore == nil {
+			logger.Warnf("restoring %s %s: get failed", name, key)
+			return
+		}
+		restore.Data = orig
+		if err := kubeClient.Update(context.Background(), restore); err != nil {
+			logger.Warnf("restoring %s %s: %v", name, key, err)
+		}
+	}, nil
+}
+
+// stripTSNetZone removes a previously-appended `ts.net:53 { ... }` server block
+// from a Corefile.
+func stripTSNetZone(corefile string) string {
+	idx := strings.Index(corefile, "ts.net:53 {")
+	if idx == -1 {
+		return corefile
+	}
+	rest := corefile[idx:]
+	end := strings.Index(rest, "\n}")
+	if end == -1 {
+		return corefile[:idx]
+	}
+	return corefile[:idx] + rest[end+len("\n}"):]
+}
+
 // forwardLocalPortToPod sets up port forwarding to the specified Pod and remote port.
 // It runs until the provided ctx is done.
 func forwardLocalPortToPod(ctx context.Context, logger *zap.SugaredLogger, cfg *rest.Config, ns, podName string, port int) error {
@@ -897,7 +1041,7 @@ func createOrUpdate(ctx context.Context, cl client.Client, obj client.Object) er
 func detectClusterIPFamilies(ctx context.Context, logger *zap.SugaredLogger, cl client.Client) error {
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      generateName("ipfamily-probe"),
+			Name:      "ipfamily-probe",
 			Namespace: ns,
 		},
 		Spec: corev1.ServiceSpec{
@@ -933,46 +1077,6 @@ func detectClusterIPFamilies(ctx context.Context, logger *zap.SugaredLogger, cl
 	return nil
 }
 
-// tailnetTarget holds the FQDN, IPv4, and IPv6 addresses of the tailnet
-// HTTP server used as the egress target.
-type tailnetTarget struct {
-	fqdn, ipv4, ipv6 string
-}
-
-// startTailnetHTTPServer starts an HTTP server that returns the tailnet FQDN, IPv4,
-// and IPv6 addresses of the created node. Used as an egress target in tests.
-func startTailnetHTTPServer(ctx context.Context, cl *tsnet.Server) (tailnetTarget, error) {
-	ln, err := cl.Listen("tcp", ":80")
-	if err != nil {
-		return tailnetTarget{}, fmt.Errorf("failed to listen on tailnet: %w", err)
-	}
-	go func() {
-		if err := http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusOK)
-		})); err != nil && !errors.Is(err, net.ErrClosed) {
-			log.Printf("tailnet HTTP server exited: %v", err)
-		}
-	}()
-
-	lc, err := cl.LocalClient()
-	if err != nil {
-		return tailnetTarget{}, fmt.Errorf("failed to get local client: %w", err)
-	}
-	status, err := lc.StatusWithoutPeers(ctx)
-	if err != nil {
-		return tailnetTarget{}, fmt.Errorf("failed to get status: %w", err)
-	}
-	target := tailnetTarget{fqdn: strings.TrimSuffix(status.Self.DNSName, ".")}
-	for _, ip := range status.TailscaleIPs {
-		if ip.Is4() {
-			target.ipv4 = ip.String()
-		} else {
-			target.ipv6 = ip.String()
-		}
-	}
-	return target, nil
-}
-
 // createTailnet creates a new tailnet and returns a tailscale.Client
 // authenticated against it using the bootstrap credentials included in the
 // creation response.