From e56fac2ff9f519948729c8c4e750ec6deec3ea95 Mon Sep 17 00:00:00 2001 From: Becky Pauley Date: Sun, 29 Mar 2026 15:33:11 +0100 Subject: [PATCH] add working multitailnet ingres test and required setup logic --- cmd/k8s-operator/e2e/multitailnet_test.go | 36 +++++++++++- cmd/k8s-operator/e2e/setup.go | 69 +++++++++++++++-------- 2 files changed, 79 insertions(+), 26 deletions(-) diff --git a/cmd/k8s-operator/e2e/multitailnet_test.go b/cmd/k8s-operator/e2e/multitailnet_test.go index e54c9bfb9..693893f6e 100644 --- a/cmd/k8s-operator/e2e/multitailnet_test.go +++ b/cmd/k8s-operator/e2e/multitailnet_test.go @@ -4,7 +4,10 @@ package e2e import ( + "context" + "crypto/tls" "fmt" + "net/http" "testing" "time" @@ -16,6 +19,7 @@ tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tstest" + "tailscale.com/util/httpm" ) // TestMultiTailnet verifies that ProxyGroup resources are created in the correct Tailnet, @@ -94,6 +98,9 @@ func TestMultiTailnet(t *testing.T) { Namespace: "default", }, Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, Ports: []corev1.ServicePort{ { Name: "http", @@ -168,7 +175,8 @@ func TestMultiTailnet(t *testing.T) { } createAndCleanup(t, kubeClient, ingress) - if err := tstest.WaitFor(5*time.Minute, func() error { + var hostname string + if err := tstest.WaitFor(3*time.Minute, func() error { ing := &networkingv1.Ingress{} if err := kubeClient.Get(t.Context(), client.ObjectKey{ Namespace: "default", Name: "second-tailnet", @@ -180,10 +188,36 @@ func TestMultiTailnet(t *testing.T) { return fmt.Errorf("Ingress not ready yet") } t.Logf("Ingress hostname: %s", ing.Status.LoadBalancer.Ingress[0].Hostname) + hostname = ing.Status.LoadBalancer.Ingress[0].Hostname return nil }); err != nil { t.Fatalf("Ingress never got a hostname: %v", err) } + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: testCAs}, + DialContext: secondTNClient.Dial, + }, + } + + var resp *http.Response + if err := tstest.WaitFor(time.Minute, func() error { + req, err := http.NewRequest(httpm.GET, fmt.Sprintf("https://%s:443", hostname), nil) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + resp, err = httpClient.Do(req.WithContext(ctx)) + return err + }); err != nil { + t.Fatalf("error trying to reach Ingress: %v", err) + } + + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status: %v", resp.StatusCode) + } + // TODO: cleanup second tailnet } diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index afcc019c0..67d29734d 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -74,7 +74,8 @@ secondTSClient *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet). secondClientID string // OAuth client_id for second tailnet. secondClientSecret string // OAuth client_secret for second tailnet. - tnClient *tsnet.Server // For testing real tailnet traffic. + tnClient *tsnet.Server // For testing real tailnet trafficon primary tailnet. + secondTNClient *tsnet.Server // For testing real tailnet traffic on second tailnet. restCfg *rest.Config // For constructing a client-go client if necessary. kubeClient client.WithWatch // For k8s API calls. clusterLoginServer string @@ -392,30 +393,29 @@ func runTests(m *testing.M) (int, error) { tsClient = tailscale.NewClient("-", tailscale.APIKey(tk.AccessToken)) // tsClient.BaseURL = "http://localhost:31544" - // secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET") - // if clientSecret == "" { - // return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") - // } - // // Format is "tskey-client--". - // parts = strings.Split(clientSecret, "-") - // if len(parts) != 4 { - // return 0, fmt.Errorf("SECOND_TS_API_CLIENT_SECRET is not valid") - // } - // secondClientID = parts[2] - // secondCredentials := clientcredentials.Config{ - // ClientID: secondClientID, - // ClientSecret: secondClientSecret, - // TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL), - // Scopes: []string{"auth_keys"}, - // } - // secondTk, err := secondCredentials.Token(ctx) - // if err != nil { - // return 0, fmt.Errorf("failed to get OAuth token: %w", err) - // } - // // An access token will last for an hour which is plenty of time for - // // the tests to run. No need for token refresh logic. - // secondTSClient = tailscale.NewClient("-", tailscale.APIKey(secondTk.AccessToken)) - // secondTSClient.BaseURL = "http://localhost:31544" + secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET") + if secondClientSecret == "" { + return 0, fmt.Errorf("must use --devcontrol or set SECOND_TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") + } + // Format is "tskey-client--". + parts = strings.Split(secondClientSecret, "-") + if len(parts) != 4 { + return 0, fmt.Errorf("SECOND_TS_API_CLIENT_SECRET is not valid") + } + secondClientID = parts[2] + secondCredentials := clientcredentials.Config{ + ClientID: secondClientID, + ClientSecret: secondClientSecret, + TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL), + Scopes: []string{"auth_keys"}, + } + secondTk, err := secondCredentials.Token(ctx) + if err != nil { + return 0, fmt.Errorf("failed to get OAuth token: %w", err) + } + // An access token will last for an hour which is plenty of time for + // the tests to run. No need for token refresh logic. + secondTSClient = tailscale.NewClient("-", tailscale.APIKey(secondTk.AccessToken)) } var ossTag string @@ -548,6 +548,12 @@ func runTests(m *testing.M) (int, error) { } defer tsClient.DeleteKey(context.Background(), authKeyMeta.ID) + secondauthKey, secondauthKeyMeta, err := secondTSClient.CreateKey(ctx, caps) + if err != nil { + return 0, err + } + defer secondTSClient.DeleteKey(context.Background(), secondauthKeyMeta.ID) + tnClient = &tsnet.Server{ ControlURL: tsClient.BaseURL, Hostname: "test-proxy", @@ -561,6 +567,19 @@ func runTests(m *testing.M) (int, error) { } defer tnClient.Close() + secondTNClient = &tsnet.Server{ + ControlURL: secondTSClient.BaseURL, + Hostname: "test-proxy", + Ephemeral: true, + Store: &mem.Store{}, + AuthKey: secondauthKey, + } + _, err = secondTNClient.Up(ctx) + if err != nil { + return 0, err + } + defer secondTNClient.Close() + return m.Run(), nil }