add working multitailnet ingres test and required setup logic

cmd/k8s-operator/e2e/multitailnet_test.go

@@ -4,7 +4,10 @@
 package e2e
 
 import (
+	"context"
+	"crypto/tls"
 	"fmt"
+	"net/http"
 	"testing"
 	"time"
 
@@ -16,6 +19,7 @@
 
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/tstest"
+	"tailscale.com/util/httpm"
 )
 
 // TestMultiTailnet verifies that ProxyGroup resources are created in the correct Tailnet,
@@ -94,6 +98,9 @@ func TestMultiTailnet(t *testing.T) {
 			Namespace: "default",
 		},
 		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"app.kubernetes.io/name": "nginx",
+			},
 			Ports: []corev1.ServicePort{
 				{
 					Name: "http",
@@ -168,7 +175,8 @@ func TestMultiTailnet(t *testing.T) {
 	}
 	createAndCleanup(t, kubeClient, ingress)
 
-	if err := tstest.WaitFor(5*time.Minute, func() error {
+	var hostname string
+	if err := tstest.WaitFor(3*time.Minute, func() error {
 		ing := &networkingv1.Ingress{}
 		if err := kubeClient.Get(t.Context(), client.ObjectKey{
 			Namespace: "default", Name: "second-tailnet",
@@ -180,10 +188,36 @@ func TestMultiTailnet(t *testing.T) {
 			return fmt.Errorf("Ingress not ready yet")
 		}
 		t.Logf("Ingress hostname: %s", ing.Status.LoadBalancer.Ingress[0].Hostname)
+		hostname = ing.Status.LoadBalancer.Ingress[0].Hostname
 		return nil
 	}); err != nil {
 		t.Fatalf("Ingress never got a hostname: %v", err)
 	}
 
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{RootCAs: testCAs},
+			DialContext:     secondTNClient.Dial,
+		},
+	}
+
+	var resp *http.Response
+	if err := tstest.WaitFor(time.Minute, func() error {
+		req, err := http.NewRequest(httpm.GET, fmt.Sprintf("https://%s:443", hostname), nil)
+		if err != nil {
+			return err
+		}
+		ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+		defer cancel()
+		resp, err = httpClient.Do(req.WithContext(ctx))
+		return err
+	}); err != nil {
+		t.Fatalf("error trying to reach Ingress: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("unexpected status: %v", resp.StatusCode)
+	}
+
 	// TODO: cleanup second tailnet
 }

cmd/k8s-operator/e2e/setup.go

@@ -74,7 +74,8 @@
 	secondTSClient     *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet).
 	secondClientID     string            // OAuth client_id for second tailnet.
 	secondClientSecret string            // OAuth client_secret for second tailnet.
-	tnClient           *tsnet.Server     // For testing real tailnet traffic.
+	tnClient           *tsnet.Server     // For testing real tailnet trafficon primary tailnet.
+	secondTNClient     *tsnet.Server     // For testing real tailnet traffic on second tailnet.
 	restCfg            *rest.Config      // For constructing a client-go client if necessary.
 	kubeClient         client.WithWatch  // For k8s API calls.
 	clusterLoginServer string
@@ -392,30 +393,29 @@ func runTests(m *testing.M) (int, error) {
 		tsClient = tailscale.NewClient("-", tailscale.APIKey(tk.AccessToken))
 		// tsClient.BaseURL = "http://localhost:31544"
 
-		// secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET")
-		// if clientSecret == "" {
-		// 	return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator")
-		// }
-		// // Format is "tskey-client-<id>-<random>".
-		// parts = strings.Split(clientSecret, "-")
-		// if len(parts) != 4 {
-		// 	return 0, fmt.Errorf("SECOND_TS_API_CLIENT_SECRET is not valid")
-		// }
-		// secondClientID = parts[2]
-		// secondCredentials := clientcredentials.Config{
-		// 	ClientID:     secondClientID,
-		// 	ClientSecret: secondClientSecret,
-		// 	TokenURL:     fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL),
-		// 	Scopes:       []string{"auth_keys"},
-		// }
-		// secondTk, err := secondCredentials.Token(ctx)
-		// if err != nil {
-		// 	return 0, fmt.Errorf("failed to get OAuth token: %w", err)
-		// }
-		// // An access token will last for an hour which is plenty of time for
-		// // the tests to run. No need for token refresh logic.
-		// secondTSClient = tailscale.NewClient("-", tailscale.APIKey(secondTk.AccessToken))
-		// secondTSClient.BaseURL = "http://localhost:31544"
+		secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET")
+		if secondClientSecret == "" {
+			return 0, fmt.Errorf("must use --devcontrol or set SECOND_TS_API_CLIENT_SECRET to an OAuth client suitable for the operator")
+		}
+		// Format is "tskey-client-<id>-<random>".
+		parts = strings.Split(secondClientSecret, "-")
+		if len(parts) != 4 {
+			return 0, fmt.Errorf("SECOND_TS_API_CLIENT_SECRET is not valid")
+		}
+		secondClientID = parts[2]
+		secondCredentials := clientcredentials.Config{
+			ClientID:     secondClientID,
+			ClientSecret: secondClientSecret,
+			TokenURL:     fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL),
+			Scopes:       []string{"auth_keys"},
+		}
+		secondTk, err := secondCredentials.Token(ctx)
+		if err != nil {
+			return 0, fmt.Errorf("failed to get OAuth token: %w", err)
+		}
+		// An access token will last for an hour which is plenty of time for
+		// the tests to run. No need for token refresh logic.
+		secondTSClient = tailscale.NewClient("-", tailscale.APIKey(secondTk.AccessToken))
 	}
 
 	var ossTag string
@@ -548,6 +548,12 @@ func runTests(m *testing.M) (int, error) {
 	}
 	defer tsClient.DeleteKey(context.Background(), authKeyMeta.ID)
 
+	secondauthKey, secondauthKeyMeta, err := secondTSClient.CreateKey(ctx, caps)
+	if err != nil {
+		return 0, err
+	}
+	defer secondTSClient.DeleteKey(context.Background(), secondauthKeyMeta.ID)
+
 	tnClient = &tsnet.Server{
 		ControlURL: tsClient.BaseURL,
 		Hostname:   "test-proxy",
@@ -561,6 +567,19 @@ func runTests(m *testing.M) (int, error) {
 	}
 	defer tnClient.Close()
 
+	secondTNClient = &tsnet.Server{
+		ControlURL: secondTSClient.BaseURL,
+		Hostname:   "test-proxy",
+		Ephemeral:  true,
+		Store:      &mem.Store{},
+		AuthKey:    secondauthKey,
+	}
+	_, err = secondTNClient.Up(ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer secondTNClient.Close()
+
 	return m.Run(), nil
 }