refactor test setup for oauth credentials

2026-04-03 22:25:27 -04:00 · 2026-03-30 10:42:53 +01:00
parent a9ca71d694
commit 520febe9f7
1 changed files with 88 additions and 79 deletions
--- a/cmd/k8s-operator/e2e/setup.go
+++ b/cmd/k8s-operator/e2e/setup.go
@@ -32,7 +32,6 @@
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"go.uber.org/zap"
-	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
@@ -72,10 +71,8 @@

 var (
 	tsClient           *tailscale.Client // For API calls to control.
+	tnClient           *tsnet.Server     // For testing real tailnet traffic on primary tailnet.
 	secondTSClient     *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet).
-	secondClientID     string            // OAuth client_id for second tailnet.
-	secondClientSecret string            // OAuth client_secret for second tailnet.
-	tnClient           *tsnet.Server     // For testing real tailnet trafficon primary tailnet.
 	secondTNClient     *tsnet.Server     // For testing real tailnet traffic on second tailnet.
 	restCfg            *rest.Config      // For constructing a client-go client if necessary.
 	kubeClient         client.WithWatch  // For k8s API calls.
@@ -164,6 +161,9 @@ func runTests(m *testing.M) (int, error) {
 		caPaths                []string // Extra CA cert file paths to add to images.

 		certsDir string = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images.
+
+		secondClientID     string // OAuth client_id for second tailnet.
+		secondClientSecret string // OAuth client_secret for second tailnet.
 	)
 	if *fDevcontrol {
 		// Deploy pebble and get its certs.
@@ -304,28 +304,32 @@ func runTests(m *testing.M) (int, error) {
 		clientSecret = key.Key
 		logger.Info("set Oauth credentials for primary tailnet")

-		// Create second tailnet.
-		secondClientCreds, err := createTailnet(tsClient.BaseURL, apiKeyData.APIKey)
+		// Create second tailnet. The bootstrap credentials returned have all permissions-
+		// they are used only to create an OAuth client for the k8s-operator
+		bootstrapID, bootstrapSecret, err := createTailnet(ctx, tsClient)
 		if err != nil {
 			return 0, fmt.Errorf("failed to create second tailnet: %w", err)
 		}
-
-		// Set up client for second tailnet- todo -get magic dns name from repsonse too?
-		secondClientID = secondClientCreds.ClientID
-		secondClientSecret = secondClientCreds.ClientSecret
-		source := secondClientCreds.TokenSource(ctx)
-		httpClient := oauth2.NewClient(ctx, source)
-		secondTSClient = tailscale.NewClient("-", nil)
-		secondTSClient.UserAgent = "e2e"
-		secondTSClient.HTTPClient = httpClient
-		secondTSClient.BaseURL = "http://localhost:31544"
-
-		logger.Infof("OAUTH_CLIENT_ID=%s", secondClientID)
-		logger.Infof("OAUTH_CLIENT_SECRET=%s", secondClientSecret)
-
+		bootstrapClient, err := oauthTSClient(ctx, "http://localhost:31544", bootstrapID, bootstrapSecret)
+		if err != nil {
+			return 0, fmt.Errorf("failed to set up bootstrap client for second tailnet: %w", err)
+		}
+		// Set HTTPS on second tailnet.
+		req, _ = http.NewRequestWithContext(ctx, http.MethodPatch, bootstrapClient.BuildTailnetURL("settings"),
+			bytes.NewBufferString(`{"httpsEnabled": true}`))
+		resp, err = bootstrapClient.Do(req)
+		if err != nil {
+			return 0, fmt.Errorf("failed to enable HTTPS: %w", err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			b, _ := io.ReadAll(resp.Body)
+			return 0, fmt.Errorf("HTTP %d enabling HTTPS: %s", resp.StatusCode, string(b))
+		}
+		logger.Info("HTTPS settings configured for second tailnet")
 		// Set ACLs for second tailnet.
-		req, _ = http.NewRequestWithContext(ctx, "POST", secondTSClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs))
-		resp, err = secondTSClient.Do(req)
+		req, _ = http.NewRequestWithContext(ctx, http.MethodPost, bootstrapClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs))
+		resp, err = bootstrapClient.Do(req)
 		if err != nil {
 			return 0, fmt.Errorf("failed to set ACLs: %w", err)
 		}
@@ -334,38 +338,44 @@ func runTests(m *testing.M) (int, error) {
 			b, _ := io.ReadAll(resp.Body)
 			return 0, fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b))
 		}
+		logger.Info("set ACLs for second tailnet")

-		reqbody := []byte(`{
-			"keyType": "client",
-			"scopes": [
-				"all"
-			],
-			"tags": ["tag:k8s-operator"]
-		}`)
-		req, _ = http.NewRequestWithContext(ctx, "PUT", secondTSClient.BuildTailnetURL("keys", secondClientID), bytes.NewReader(reqbody))
-		resp, err = secondTSClient.Do(req)
+		// Create an OAuth client for the second tailnet with the same
+		// scopes and tag as the primary tailnet's k8s-operator.
+		reqBody, err = json.Marshal(map[string]any{
+			"keyType":     "client",
+			"scopes":      []string{"auth_keys", "devices:core", "services"},
+			"tags":        []string{"tag:k8s-operator"},
+			"description": "k8s-operator client for e2e tests",
+		})
 		if err != nil {
-			return 0, fmt.Errorf("failed to set oauth scopes: %w", err)
+			return 0, fmt.Errorf("failed to marshal OAuth client creation request: %w", err)
+		}
+		req, _ = http.NewRequestWithContext(ctx, http.MethodPost, bootstrapClient.BuildTailnetURL("keys"), bytes.NewReader(reqBody))
+		resp, err = bootstrapClient.Do(req)
+		if err != nil {
+			return 0, fmt.Errorf("failed to create OAuth client for second tailnet: %w", err)
 		}
 		defer resp.Body.Close()
 		if resp.StatusCode != http.StatusOK {
 			b, _ := io.ReadAll(resp.Body)
-			return 0, fmt.Errorf("HTTP %d setting oauth scopes: %s", resp.StatusCode, string(b))
+			return 0, fmt.Errorf("HTTP %d creating OAuth client for second tailnet: %s", resp.StatusCode, string(b))
 		}
-		logger.Infof("Oauth scopes configured")
+		var secondKey struct {
+			ID  string `json:"id"`
+			Key string `json:"key"`
+		}
+		if err := json.NewDecoder(resp.Body).Decode(&secondKey); err != nil {
+			return 0, fmt.Errorf("failed to decode OAuth client creation response: %w", err)
+		}
+		secondClientID = secondKey.ID
+		secondClientSecret = secondKey.Key
+		logger.Info("set OAuth credentials for second tailnet")

-		// Set HTTPS on second Tailnet
-		req, _ = http.NewRequestWithContext(ctx, "PATCH", secondTSClient.BuildTailnetURL("settings"), bytes.NewBuffer([]byte(`{"httpsEnabled": true}`)))
-		resp, err = secondTSClient.Do(req)
+		secondTSClient, err = oauthTSClient(ctx, "http://localhost:31544", secondClientID, secondClientSecret)
 		if err != nil {
-			return 0, fmt.Errorf("failed to enable HTTPS: %w", err)
+			return 0, fmt.Errorf("failed to set up second tailnet client: %w", err)
 		}
-		defer resp.Body.Close()
-		if resp.StatusCode != http.StatusOK {
-			b, _ := io.ReadAll(resp.Body)
-			return 0, fmt.Errorf("HTTP %d enabling https: %s", resp.StatusCode, string(b))
-		}
-		logger.Infof("HTTPS settings configured")

 	} else {
 		clientSecret = os.Getenv("TS_API_CLIENT_SECRET")
@@ -378,19 +388,10 @@ func runTests(m *testing.M) (int, error) {
 			return 0, fmt.Errorf("TS_API_CLIENT_SECRET is not valid")
 		}
 		clientID = parts[2]
-		credentials := clientcredentials.Config{
-			ClientID:     clientID,
-			ClientSecret: clientSecret,
-			TokenURL:     fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL),
-			Scopes:       []string{"auth_keys"},
-		}
-		tk, err := credentials.Token(ctx)
+		tsClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, clientID, clientSecret)
 		if err != nil {
-			return 0, fmt.Errorf("failed to get OAuth token: %w", err)
+			return 0, fmt.Errorf("failed to set up primary tailnet client: %w", err)
 		}
-		// An access token will last for an hour which is plenty of time for
-		// the tests to run. No need for token refresh logic.
-		tsClient = tailscale.NewClient("-", tailscale.APIKey(tk.AccessToken))

 		secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET")
 		if secondClientSecret == "" {
@@ -402,19 +403,10 @@ func runTests(m *testing.M) (int, error) {
 			return 0, fmt.Errorf("SECOND_TS_API_CLIENT_SECRET is not valid")
 		}
 		secondClientID = parts[2]
-		secondCredentials := clientcredentials.Config{
-			ClientID:     secondClientID,
-			ClientSecret: secondClientSecret,
-			TokenURL:     fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL),
-			Scopes:       []string{"auth_keys"},
-		}
-		secondTk, err := secondCredentials.Token(ctx)
+		secondTSClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, secondClientID, secondClientSecret)
 		if err != nil {
-			return 0, fmt.Errorf("failed to get OAuth token: %w", err)
+			return 0, fmt.Errorf("failed to set up second tailnet client: %w", err)
 		}
-		// An access token will last for an hour which is plenty of time for
-		// the tests to run. No need for token refresh logic.
-		secondTSClient = tailscale.NewClient("-", tailscale.APIKey(secondTk.AccessToken))
 	}

 	var ossTag string
@@ -886,22 +878,25 @@ func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts
 	return nil
 }

-func createTailnet(baseURL, apiKey string) (clientcredentials.Config, error) {
+func createTailnet(ctx context.Context, cl *tailscale.Client) (clientID, clientSecret string, err error) {
 	tailnetName := fmt.Sprintf("second-tailnet-%d", time.Now().Unix())
 	body, err := json.Marshal(map[string]any{"displayName": tailnetName})
 	if err != nil {
-		return clientcredentials.Config{}, err
+		return "", "", err
 	}
-	req, _ := http.NewRequest("POST", baseURL+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body))
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
-	resp, err := http.DefaultClient.Do(req)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		cl.BaseURL+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body))
 	if err != nil {
-		return clientcredentials.Config{}, err
+		return "", "", err
+	}
+	resp, err := cl.Do(req)
+	if err != nil {
+		return "", "", err
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		b, _ := io.ReadAll(resp.Body)
-		return clientcredentials.Config{}, fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b))
+		return "", "", fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b))
 	}
 	var result struct {
 		OauthClient struct {
@@ -910,11 +905,25 @@ func createTailnet(baseURL, apiKey string) (clientcredentials.Config, error) {
 		} `json:"oauthClient"`
 	}
 	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-		return clientcredentials.Config{}, fmt.Errorf("failed to decode response: %w", err)
+		return "", "", fmt.Errorf("failed to decode response: %w", err)
 	}
-	return clientcredentials.Config{
-		ClientID:     result.OauthClient.ID,
-		ClientSecret: result.OauthClient.Secret,
-		TokenURL:     baseURL + "/api/v2/oauth/token",
-	}, nil
+	return result.OauthClient.ID, result.OauthClient.Secret, nil
+}
+
+// oauthTSClient exchanges OAuth client credentials for an access token and
+// returns a tailscale.Client configured to use it. The token is valid for
+// one hour, which is sufficient for a single test run.
+func oauthTSClient(ctx context.Context, baseURL, clientID, clientSecret string) (*tailscale.Client, error) {
+	cfg := clientcredentials.Config{
+		ClientID:     clientID,
+		ClientSecret: clientSecret,
+		TokenURL:     fmt.Sprintf("%s/api/v2/oauth/token", baseURL),
+	}
+	tk, err := cfg.Token(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get OAuth token for client %q: %w", clientID, err)
+	}
+	c := tailscale.NewClient("-", tailscale.APIKey(tk.AccessToken))
+	c.BaseURL = baseURL
+	return c, nil
 }