diff --git a/cmd/k8s-operator/e2e/egress_test.go b/cmd/k8s-operator/e2e/egress_test.go index 9f0a530ce..d6c772246 100644 --- a/cmd/k8s-operator/e2e/egress_test.go +++ b/cmd/k8s-operator/e2e/egress_test.go @@ -4,13 +4,19 @@ package e2e import ( + "errors" "fmt" + "log" + "net" + "net/http" + "strings" "testing" "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "tailscale.com/client/tailscale/v2" kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tstest" @@ -18,6 +24,12 @@ const egressPort = 80 +// tailnetTarget holds the FQDN, IPv4, and IPv6 addresses of a tailnet node +// running an HTTP server, for use as an egress target in tests. +type tailnetTarget struct { + fqdn, ipv4, ipv6 string +} + // See [TestMain] for test requirements. func TestEgress(t *testing.T) { if tnClient == nil { @@ -25,16 +37,18 @@ func TestEgress(t *testing.T) { } t.Parallel() + target := startEgressTarget(t, tsClient) + t.Run("IPv4", func(t *testing.T) { if !clusterIPv4Support { t.Skip("cluster does not support IPv4") } svc := egressService(generateName("test-egress"), map[string]string{ - "tailscale.com/tailnet-ip": tnTarget.ipv4, + "tailscale.com/tailnet-ip": target.ipv4, }) createAndCleanup(t, kubeClient, svc) waitForEgress(t, svc.Name, kube.SvcIsReady) - testEgressIsReachable(t, ns, svc.Name) + requireTargetIsReachable(t, egressURL(svc.Name)) }) t.Run("IPv6", func(t *testing.T) { @@ -42,20 +56,20 @@ func TestEgress(t *testing.T) { t.Skip("cluster does not support IPv6") } svc := egressService(generateName("test-egress"), map[string]string{ - "tailscale.com/tailnet-ip": tnTarget.ipv6, + "tailscale.com/tailnet-ip": target.ipv6, }) createAndCleanup(t, kubeClient, svc) waitForEgress(t, svc.Name, kube.SvcIsReady) - testEgressIsReachable(t, ns, svc.Name) + requireTargetIsReachable(t, egressURL(svc.Name)) }) t.Run("FQDN", func(t *testing.T) { svc := egressService(generateName("test-egress"), map[string]string{ - "tailscale.com/tailnet-fqdn": tnTarget.fqdn, + "tailscale.com/tailnet-fqdn": target.fqdn, }) createAndCleanup(t, kubeClient, svc) waitForEgress(t, svc.Name, kube.SvcIsReady) - testEgressIsReachable(t, ns, svc.Name) + requireTargetIsReachable(t, egressURL(svc.Name)) }) } @@ -66,6 +80,8 @@ func TestHAEgress(t *testing.T) { } t.Parallel() + target := startEgressTarget(t, tsClient) + pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: generateName("egress"), @@ -81,12 +97,12 @@ func TestHAEgress(t *testing.T) { t.Skip("cluster does not support IPv4") } svc := egressService(generateName("test-egress"), map[string]string{ - "tailscale.com/tailnet-ip": tnTarget.ipv4, + "tailscale.com/tailnet-ip": target.ipv4, "tailscale.com/proxy-group": pg.Name, }) createAndCleanup(t, kubeClient, svc) waitForEgress(t, svc.Name, pgEgressReady) - testEgressIsReachable(t, ns, svc.Name) + requireTargetIsReachable(t, egressURL(svc.Name)) }) t.Run("IPv6", func(t *testing.T) { @@ -94,22 +110,24 @@ func TestHAEgress(t *testing.T) { t.Skip("cluster does not support IPv6") } svc := egressService(generateName("test-egress"), map[string]string{ - "tailscale.com/tailnet-ip": tnTarget.ipv6, + "tailscale.com/tailnet-ip": target.ipv6, "tailscale.com/proxy-group": pg.Name, }) createAndCleanup(t, kubeClient, svc) waitForEgress(t, svc.Name, pgEgressReady) - testEgressIsReachable(t, ns, svc.Name) + requireTargetIsReachable(t, egressURL(svc.Name)) }) t.Run("FQDN", func(t *testing.T) { svc := egressService(generateName("test-egress"), map[string]string{ - "tailscale.com/tailnet-fqdn": tnTarget.fqdn, + "tailscale.com/tailnet-fqdn": target.fqdn, "tailscale.com/proxy-group": pg.Name, }) createAndCleanup(t, kubeClient, svc) waitForEgress(t, svc.Name, pgEgressReady) - testEgressIsReachable(t, ns, svc.Name) + // Test the egress target is reachable via both its in-cluster Service name, and its Magic DNS name. + requireTargetIsReachable(t, egressURL(svc.Name)) + requireTargetIsReachable(t, fmt.Sprintf("http://%s:%d", target.fqdn, egressPort)) }) } @@ -134,13 +152,51 @@ func TestHAEgressMultiTailnet(t *testing.T) { t.Fatalf("verifying ProxyGroup %s is registered to the correct tailnet: %v", pg.Name, err) } + target := startEgressTarget(t, secondTSClient) svc := egressService(generateName("test-egress"), map[string]string{ - "tailscale.com/tailnet-fqdn": secondTNTarget.fqdn, + "tailscale.com/tailnet-fqdn": target.fqdn, "tailscale.com/proxy-group": pg.Name, }) createAndCleanup(t, kubeClient, svc) waitForEgress(t, svc.Name, pgEgressReady) - testEgressIsReachable(t, ns, svc.Name) + // Test the egress target is reachable via both its in-cluster Service name, and its Magic DNS name. + requireTargetIsReachable(t, egressURL(svc.Name)) + requireTargetIsReachable(t, fmt.Sprintf("http://%s:%d", target.fqdn, egressPort)) +} + +func startEgressTarget(t *testing.T, cl *tailscale.Client) tailnetTarget { + t.Helper() + srv := newTailnetNode(t, cl, generateName("test-egress-target")) + + ln, err := srv.Listen("tcp", fmt.Sprintf(":%d", egressPort)) + if err != nil { + t.Fatalf("listening on egress target: %v", err) + } + go func() { + if err := http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })); err != nil && !errors.Is(err, net.ErrClosed) { + log.Printf("egress target HTTP server exited: %v", err) + } + }() + + lc, err := srv.LocalClient() + if err != nil { + t.Fatalf("getting egress target local client: %v", err) + } + status, err := lc.StatusWithoutPeers(t.Context()) + if err != nil { + t.Fatalf("getting egress target status: %v", err) + } + target := tailnetTarget{fqdn: strings.TrimSuffix(status.Self.DNSName, ".")} + for _, ip := range status.TailscaleIPs { + if ip.Is4() { + target.ipv4 = ip.String() + } else { + target.ipv6 = ip.String() + } + } + return target } func egressService(name string, annotations map[string]string) *corev1.Service { @@ -169,6 +225,10 @@ func pgEgressReady(svc *corev1.Service) bool { return cond != nil && cond.Status == metav1.ConditionTrue } +func egressURL(svcName string) string { + return fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", svcName, ns, egressPort) +} + func waitForEgress(t *testing.T, svcName string, ready func(*corev1.Service) bool) { t.Helper() if err := tstest.WaitFor(5*time.Minute, func() error { @@ -185,43 +245,3 @@ func waitForEgress(t *testing.T, svcName string, ready func(*corev1.Service) boo t.Fatalf("error waiting for Service %s to become ready: %v", svcName, err) } } - -func testEgressIsReachable(t *testing.T, namespace, svcName string) { - t.Helper() - url := fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", svcName, namespace, egressPort) - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: generateName("curl"), - Namespace: namespace, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - Containers: []corev1.Container{ - { - Name: "curl", - Image: "curlimages/curl", - Command: []string{"sh", "-c", fmt.Sprintf( - `for i in $(seq 1 10); do `+ - `code=$(curl -s -o /dev/null -w "%%{http_code}" --max-time 5 %q); `+ - `[ "$code" = "200" ] && exit 0; sleep 2; done; exit 1`, url)}, - }, - }, - }, - } - createAndCleanup(t, kubeClient, pod) - - if err := tstest.WaitFor(2*time.Minute, func() error { - p := &corev1.Pod{ObjectMeta: objectMeta(namespace, pod.Name)} - if err := get(t.Context(), kubeClient, p); err != nil { - return err - } - if p.Status.Phase == corev1.PodSucceeded { - t.Logf("curl pod %s succeeded", pod.Name) - return nil - } - return fmt.Errorf("curl pod %s phase: %s", pod.Name, p.Status.Phase) - }); err != nil { - t.Fatalf("egress service %s/%s not reachable: %v", - namespace, svcName, err) - } -} diff --git a/cmd/k8s-operator/e2e/helpers.go b/cmd/k8s-operator/e2e/helpers_test.go similarity index 57% rename from cmd/k8s-operator/e2e/helpers.go rename to cmd/k8s-operator/e2e/helpers_test.go index ceee3c3f6..c797f933d 100644 --- a/cmd/k8s-operator/e2e/helpers.go +++ b/cmd/k8s-operator/e2e/helpers_test.go @@ -4,6 +4,7 @@ package e2e import ( + "context" "crypto/rand" "crypto/tls" "fmt" @@ -13,8 +14,10 @@ "time" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - + "tailscale.com/client/tailscale/v2" + "tailscale.com/ipn/store/mem" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/tsnet" @@ -25,6 +28,72 @@ func generateName(prefix string) string { return fmt.Sprintf("%s-%s", prefix, strings.ToLower(rand.Text())) } +func requireTargetIsReachable(t *testing.T, url string) { + t.Helper() + + volumes, mounts, cacertFlag := certVolumesForURL(url) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName("curl"), + Namespace: ns, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Volumes: volumes, + Containers: []corev1.Container{ + { + Name: "curl", + Image: "curlimages/curl", + VolumeMounts: mounts, + Command: []string{"sh", "-c", fmt.Sprintf( + `for i in $(seq 1 40); do `+ + `code=$(curl -s %s-o /dev/null -w "%%{http_code}" --max-time 5 %q); `+ + `[ "$code" = "200" ] && exit 0; sleep 2; done; exit 1`, cacertFlag, url)}, + }, + }, + }, + } + createAndCleanup(t, kubeClient, pod) + + if err := tstest.WaitFor(5*time.Minute, func() error { + p := &corev1.Pod{ObjectMeta: objectMeta(ns, pod.Name)} + if err := get(t.Context(), kubeClient, p); err != nil { + return err + } + if p.Status.Phase == corev1.PodSucceeded { + t.Logf("curl pod %s succeeded", pod.Name) + return nil + } + if p.Status.Phase == corev1.PodFailed { + t.Fatalf("%s not reachable in-cluster: curl pod %s failed", url, pod.Name) + } + return fmt.Errorf("curl pod %s phase: %s", pod.Name, p.Status.Phase) + }); err != nil { + t.Fatalf("%s not reachable in-cluster: %v", url, err) + } +} + +// certVolumesForURL returns the Pod volume, VolumeMount, and curl "--cacert" +// argument needed to verify an HTTPS url against the test CAs published in the testCAsConfigMap. +func certVolumesForURL(url string) ([]corev1.Volume, []corev1.VolumeMount, string) { + if !strings.HasPrefix(url, "https://") { + return nil, nil, "" + } + const mountPath = "/etc/test-cas" + volumes := []corev1.Volume{{ + Name: "test-cas", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: testCAsConfigMap}, + }, + }, + }} + mounts := []corev1.VolumeMount{{Name: "test-cas", MountPath: mountPath, ReadOnly: true}} + cacertFlag := "--cacert " + mountPath + "/" + testCAsConfigMapKey + " " + return volumes, mounts, cacertFlag +} + // newHTTPClient returns a HTTP client for the given tailnet client. // When running against devcontrol, trusts Pebble testCAs. Otherwise, // trusts Let's Encrypt staging testCA. @@ -130,3 +199,29 @@ func verifyProxyGroupTailnet(t *testing.T, pg *tsapi.ProxyGroup, cl *tsnet.Serve } return nil } + +func newTailnetNode(t *testing.T, cl *tailscale.Client, hostname string) *tsnet.Server { + t.Helper() + caps := tailscale.KeyCapabilities{} + caps.Devices.Create.Preauthorized = true + caps.Devices.Create.Ephemeral = true + caps.Devices.Create.Tags = []string{"tag:k8s"} + authKey, err := cl.Keys().CreateAuthKey(t.Context(), tailscale.CreateKeyRequest{Capabilities: caps}) + if err != nil { + t.Fatalf("creating auth key: %v", err) + } + t.Cleanup(func() { cl.Keys().Delete(context.Background(), authKey.ID) }) + + srv := &tsnet.Server{ + ControlURL: cl.BaseURL.String(), + Hostname: hostname, + Ephemeral: true, + Store: &mem.Store{}, + AuthKey: authKey.Key, + } + if _, err := srv.Up(t.Context()); err != nil { + t.Fatalf("bringing up node: %v", err) + } + t.Cleanup(func() { srv.Close() }) + return srv +} diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go index 67dfd1b0b..9a01f9eef 100644 --- a/cmd/k8s-operator/e2e/ingress_test.go +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -227,6 +227,43 @@ func TestL7Ingress(t *testing.T) { } } +func TestL7IngressDNSConfig(t *testing.T) { + if tnClient == nil { + t.Skip("TestL7IngressDNSConfig requires a working tailnet client") + } + t.Parallel() + + nginx := nginxDeployment(ns) + createAndCleanup(t, kubeClient, nginx) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: nginx.Name, + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": nginx.Name, + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + ingress := l7Ingress(ns, nginx.Name, map[string]string{ + "tailscale.com/experimental-forward-cluster-traffic-via-ingress": "true", + }) + createAndCleanup(t, kubeClient, ingress) + hostname, err := waitForIngressHostname(t, ns, ingress.Name) + if err != nil { + t.Fatalf("error waiting for Ingress hostname: %v", err) + } + requireTargetIsReachable(t, fmt.Sprintf("https://%s:443", hostname)) +} + func TestL7HAIngress(t *testing.T) { if tnClient == nil { t.Skip("TestL7HAIngress requires a working tailnet client") diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index 93289eab4..a5f87cccf 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -11,12 +11,10 @@ "crypto/x509" _ "embed" "encoding/json" - "errors" "flag" "fmt" "io" - "log" - "net" + "maps" "net/http" "net/url" "os" @@ -68,19 +66,19 @@ ) const ( - pebbleTag = "2.8.0" - ns = "default" - tmp = "/tmp/k8s-operator-e2e" - kindClusterName = "k8s-operator-e2e" + pebbleTag = "2.8.0" + ns = "default" + tmp = "/tmp/k8s-operator-e2e" + kindClusterName = "k8s-operator-e2e" + testCAsConfigMap = "test-cas" + testCAsConfigMapKey = "test-cas.pem" ) var ( tsClient *tailscale.Client // For API calls to control. tnClient *tsnet.Server // For testing real tailnet traffic on first tailnet. - tnTarget tailnetTarget // Egress target on the first tailnet. secondTSClient *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet). secondTNClient *tsnet.Server // For testing real tailnet traffic on second tailnet. - secondTNTarget tailnetTarget // Egress target on the second tailnet. restCfg *rest.Config // For constructing a client-go client if necessary. kubeClient client.WithWatch // For k8s API calls. clusterLoginServer string @@ -187,11 +185,12 @@ func runTests(m *testing.M) (int, error) { } var ( - clientID, clientSecret string // OAuth client for the first tailnet (for the operator to use). - caPaths []string // Extra CA cert file paths to add to images. + clientID, clientSecret string // OAuth client for the first tailnet (for the operator to use). + secondClientID, secondClientSecret string // OAuth client for the second tailnet (for the operator to use). - certsDir = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images. - secondClientID, secondClientSecret string // OAuth client for the second tailnet (for the operator to use). + caPaths []string // Extra CA cert file paths to add to images. + certsDir = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images. + caPEM []byte // Used to collect and then publish test-cas ConfigMap. ) testCAs = x509.NewCertPool() if *fDevcontrol { @@ -225,6 +224,7 @@ func runTests(m *testing.M) (int, error) { if ok := testCAs.AppendCertsFromPEM(pebbleCAChain); !ok { return 0, fmt.Errorf("failed to parse pebble ca chain cert") } + caPEM = appendPEM(caPEM, pebbleMiniCACert, pebbleCAChain) if err = os.MkdirAll(certsDir, 0755); err != nil { return 0, fmt.Errorf("failed to create certs dir: %w", err) @@ -368,7 +368,7 @@ func runTests(m *testing.M) (int, error) { if ok := testCAs.AppendCertsFromPEM(leStagingRootX1); !ok { return 0, fmt.Errorf("failed to parse Let's Encrypt staging root") } - + caPEM = appendPEM(caPEM, leStagingRootX1) clientSecret = os.Getenv("TS_API_CLIENT_SECRET") if clientSecret == "" { return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") @@ -395,6 +395,19 @@ func runTests(m *testing.M) (int, error) { } } + // Publish the trustedCAs as a ConfigMap that can be used by in-cluster + // testing workloads. + if len(caPEM) > 0 { + caCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: testCAsConfigMap, Namespace: ns}, + Data: map[string]string{testCAsConfigMapKey: string(caPEM)}, + } + if err := createOrUpdate(ctx, kubeClient, caCM); err != nil { + return 0, fmt.Errorf("failed to publish test CAs ConfigMap: %w", err) + } + defer kubeClient.Delete(context.Background(), caCM) + } + var ossTag string if *fBuild { // TODO(tomhjp): proper support for --build=false and layering pebble certs on top of existing images. @@ -411,9 +424,10 @@ func runTests(m *testing.M) (int, error) { logger.Infof("using base image: %q", *fBaseImage) } ossImageToTarget := map[string]string{ - "local/k8s-operator": "publishdevoperator", - "local/tailscale": "publishdevimage", - "local/k8s-proxy": "publishdevproxy", + "local/k8s-operator": "publishdevoperator", + "local/tailscale": "publishdevimage", + "local/k8s-proxy": "publishdevproxy", + "local/k8s-nameserver": "publishdevnameserver", } for img, target := range ossImageToTarget { if err := buildImage(ctx, ossDir, img, target, ossTag, *fBaseImage, caPaths); err != nil { @@ -519,6 +533,26 @@ func runTests(m *testing.M) (int, error) { return 0, fmt.Errorf("failed to apply default ProxyClass: %w", err) } + // Leave the nameserver image unset when nothing was built so + // the operator falls back to the default. + // TODO(beckypauley): fix for other images where build is false. + nameserverImg := &tsapi.NameserverImage{} + if ossTag != "" { + nameserverImg.Repo = "local/k8s-nameserver" + nameserverImg.Tag = ossTag + } + dnsConfig, err := deployNameserver(ctx, logger, kubeClient, nameserverImg) + if err != nil { + return 0, fmt.Errorf("failed to deploy nameserver: %w", err) + } + defer kubeClient.Delete(context.Background(), dnsConfig) + + restoreClusterDNS, err := patchClusterDNS(ctx, logger, dnsConfig.Status.Nameserver.IP) + if err != nil { + return 0, fmt.Errorf("failed to patch cluster DNS: %w", err) + } + defer restoreClusterDNS() + caps := tailscale.KeyCapabilities{} caps.Devices.Create.Preauthorized = true caps.Devices.Create.Ephemeral = true @@ -548,10 +582,6 @@ func runTests(m *testing.M) (int, error) { return 0, err } defer tnClient.Close() - tnTarget, err = startTailnetHTTPServer(ctx, tnClient) - if err != nil { - return 0, fmt.Errorf("failed to start tailnet HTTP server on first tailnet: %w", err) - } secondTNClient = &tsnet.Server{ ControlURL: secondTSClient.BaseURL.String(), @@ -565,10 +595,6 @@ func runTests(m *testing.M) (int, error) { return 0, err } defer secondTNClient.Close() - secondTNTarget, err = startTailnetHTTPServer(ctx, secondTNClient) - if err != nil { - return 0, fmt.Errorf("failed to start tailnet HTTP server on second tailnet: %w", err) - } // Create the tailnet Secret in the tailscale namespace. secret := &corev1.Secret{ @@ -743,6 +769,124 @@ func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl c return nil } +// appendPEM joins PEM blobs with a newline separator so a blob lacking a +// trailing newline doesn't glue its END line to the next BEGIN line. +// Otherwise curl/OpenSSL can silently drop the later certs. +// TODO(beckypauley): avoid maintaining both caPEM and testCAs. This can then be removed. +func appendPEM(dst []byte, blobs ...[]byte) []byte { + for _, b := range blobs { + if len(b) == 0 { + continue + } + if len(dst) > 0 && dst[len(dst)-1] != '\n' { + dst = append(dst, '\n') + } + dst = append(dst, b...) + } + return dst +} + +func deployNameserver(ctx context.Context, logger *zap.SugaredLogger, cl client.Client, img *tsapi.NameserverImage) (*tsapi.DNSConfig, error) { + dc := &tsapi.DNSConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "dns"}, + Spec: tsapi.DNSConfigSpec{Nameserver: &tsapi.Nameserver{Image: img}}, + } + if err := createOrUpdate(ctx, cl, dc); err != nil { + return nil, fmt.Errorf("failed to create DNSConfig: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + ticker := time.NewTicker(time.Second * 2) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil, fmt.Errorf("timeout waiting for nameserver to be ready") + case <-ticker.C: + if err := cl.Get(ctx, client.ObjectKeyFromObject(dc), dc); err != nil { + return nil, fmt.Errorf("failed to get DNSConfig: %w", err) + } + if tsoperator.DNSCfgIsReady(dc) && dc.Status.Nameserver != nil && dc.Status.Nameserver.IP != "" { + logger.Infof("nameserver ready; Service IP %s", dc.Status.Nameserver.IP) + return dc, nil + } + logger.Info("waiting for nameserver to be ready...") + } + } +} + +func patchClusterDNS(ctx context.Context, logger *zap.SugaredLogger, nameserverIP string) (func(), error) { + if cm := getDNSConfigMap(ctx, "coredns"); cm != nil && cm.Data["Corefile"] != "" { + corefile := stripTSNetZone(cm.Data["Corefile"]) + fmt.Sprintf(` +ts.net:53 { + errors + cache 30 + forward . %s +} +`, nameserverIP) + return patchDNSConfigMap(logger, cm, "Corefile", corefile) + } + if cm := getDNSConfigMap(ctx, "kube-dns"); cm != nil { + stub, err := json.Marshal(map[string][]string{"ts.net": {nameserverIP}}) + if err != nil { + return nil, fmt.Errorf("marshalling stubDomains: %w", err) + } + return patchDNSConfigMap(logger, cm, "stubDomains", string(stub)) + } + return nil, fmt.Errorf("cluster DNS is not a patchable CoreDNS/kube-dns") +} + +func getDNSConfigMap(ctx context.Context, name string) *corev1.ConfigMap { + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "kube-system", Name: name}} + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { + return nil + } + return cm +} + +// patchDNSConfigMap updates the given Configmap and returns a closure +// for test cleanup. +func patchDNSConfigMap(logger *zap.SugaredLogger, cm *corev1.ConfigMap, key, value string) (func(), error) { + orig := maps.Clone(cm.Data) + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[key] = value + if err := kubeClient.Update(context.Background(), cm); err != nil { + return nil, fmt.Errorf("patching %s %s: %w", cm.Name, key, err) + } + logger.Infof("patched %s %s with ts.net entry", cm.Name, key) + + name := cm.Name + return func() { + restore := getDNSConfigMap(context.Background(), name) + if restore == nil { + logger.Warnf("restoring %s %s: get failed", name, key) + return + } + restore.Data = orig + if err := kubeClient.Update(context.Background(), restore); err != nil { + logger.Warnf("restoring %s %s: %v", name, key, err) + } + }, nil +} + +// stripTSNetZone removes a previously-appended `ts.net:53 { ... }` server block +// from a Corefile. +func stripTSNetZone(corefile string) string { + idx := strings.Index(corefile, "ts.net:53 {") + if idx == -1 { + return corefile + } + rest := corefile[idx:] + end := strings.Index(rest, "\n}") + if end == -1 { + return corefile[:idx] + } + return corefile[:idx] + rest[end+len("\n}"):] +} + // forwardLocalPortToPod sets up port forwarding to the specified Pod and remote port. // It runs until the provided ctx is done. func forwardLocalPortToPod(ctx context.Context, logger *zap.SugaredLogger, cfg *rest.Config, ns, podName string, port int) error { @@ -897,7 +1041,7 @@ func createOrUpdate(ctx context.Context, cl client.Client, obj client.Object) er func detectClusterIPFamilies(ctx context.Context, logger *zap.SugaredLogger, cl client.Client) error { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: generateName("ipfamily-probe"), + Name: "ipfamily-probe", Namespace: ns, }, Spec: corev1.ServiceSpec{ @@ -933,46 +1077,6 @@ func detectClusterIPFamilies(ctx context.Context, logger *zap.SugaredLogger, cl return nil } -// tailnetTarget holds the FQDN, IPv4, and IPv6 addresses of the tailnet -// HTTP server used as the egress target. -type tailnetTarget struct { - fqdn, ipv4, ipv6 string -} - -// startTailnetHTTPServer starts an HTTP server that returns the tailnet FQDN, IPv4, -// and IPv6 addresses of the created node. Used as an egress target in tests. -func startTailnetHTTPServer(ctx context.Context, cl *tsnet.Server) (tailnetTarget, error) { - ln, err := cl.Listen("tcp", ":80") - if err != nil { - return tailnetTarget{}, fmt.Errorf("failed to listen on tailnet: %w", err) - } - go func() { - if err := http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })); err != nil && !errors.Is(err, net.ErrClosed) { - log.Printf("tailnet HTTP server exited: %v", err) - } - }() - - lc, err := cl.LocalClient() - if err != nil { - return tailnetTarget{}, fmt.Errorf("failed to get local client: %w", err) - } - status, err := lc.StatusWithoutPeers(ctx) - if err != nil { - return tailnetTarget{}, fmt.Errorf("failed to get status: %w", err) - } - target := tailnetTarget{fqdn: strings.TrimSuffix(status.Self.DNSName, ".")} - for _, ip := range status.TailscaleIPs { - if ip.Is4() { - target.ipv4 = ip.String() - } else { - target.ipv6 = ip.String() - } - } - return target, nil -} - // createTailnet creates a new tailnet and returns a tailscale.Client // authenticated against it using the bootstrap credentials included in the // creation response.