diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index d36dc36f6..2c4856ade 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -32,7 +32,6 @@ "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/tarball" "go.uber.org/zap" - "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -72,10 +71,8 @@ var ( tsClient *tailscale.Client // For API calls to control. + tnClient *tsnet.Server // For testing real tailnet traffic on primary tailnet. secondTSClient *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet). - secondClientID string // OAuth client_id for second tailnet. - secondClientSecret string // OAuth client_secret for second tailnet. - tnClient *tsnet.Server // For testing real tailnet trafficon primary tailnet. secondTNClient *tsnet.Server // For testing real tailnet traffic on second tailnet. restCfg *rest.Config // For constructing a client-go client if necessary. kubeClient client.WithWatch // For k8s API calls. @@ -164,6 +161,9 @@ func runTests(m *testing.M) (int, error) { caPaths []string // Extra CA cert file paths to add to images. certsDir string = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images. + + secondClientID string // OAuth client_id for second tailnet. + secondClientSecret string // OAuth client_secret for second tailnet. ) if *fDevcontrol { // Deploy pebble and get its certs. @@ -304,28 +304,32 @@ func runTests(m *testing.M) (int, error) { clientSecret = key.Key logger.Info("set Oauth credentials for primary tailnet") - // Create second tailnet. - secondClientCreds, err := createTailnet(tsClient.BaseURL, apiKeyData.APIKey) + // Create second tailnet. The bootstrap credentials returned have all permissions- + // they are used only to create an OAuth client for the k8s-operator + bootstrapID, bootstrapSecret, err := createTailnet(ctx, tsClient) if err != nil { return 0, fmt.Errorf("failed to create second tailnet: %w", err) } - - // Set up client for second tailnet- todo -get magic dns name from repsonse too? - secondClientID = secondClientCreds.ClientID - secondClientSecret = secondClientCreds.ClientSecret - source := secondClientCreds.TokenSource(ctx) - httpClient := oauth2.NewClient(ctx, source) - secondTSClient = tailscale.NewClient("-", nil) - secondTSClient.UserAgent = "e2e" - secondTSClient.HTTPClient = httpClient - secondTSClient.BaseURL = "http://localhost:31544" - - logger.Infof("OAUTH_CLIENT_ID=%s", secondClientID) - logger.Infof("OAUTH_CLIENT_SECRET=%s", secondClientSecret) - + bootstrapClient, err := oauthTSClient(ctx, "http://localhost:31544", bootstrapID, bootstrapSecret) + if err != nil { + return 0, fmt.Errorf("failed to set up bootstrap client for second tailnet: %w", err) + } + // Set HTTPS on second tailnet. + req, _ = http.NewRequestWithContext(ctx, http.MethodPatch, bootstrapClient.BuildTailnetURL("settings"), + bytes.NewBufferString(`{"httpsEnabled": true}`)) + resp, err = bootstrapClient.Do(req) + if err != nil { + return 0, fmt.Errorf("failed to enable HTTPS: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("HTTP %d enabling HTTPS: %s", resp.StatusCode, string(b)) + } + logger.Info("HTTPS settings configured for second tailnet") // Set ACLs for second tailnet. - req, _ = http.NewRequestWithContext(ctx, "POST", secondTSClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs)) - resp, err = secondTSClient.Do(req) + req, _ = http.NewRequestWithContext(ctx, http.MethodPost, bootstrapClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs)) + resp, err = bootstrapClient.Do(req) if err != nil { return 0, fmt.Errorf("failed to set ACLs: %w", err) } @@ -334,38 +338,44 @@ func runTests(m *testing.M) (int, error) { b, _ := io.ReadAll(resp.Body) return 0, fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b)) } + logger.Info("set ACLs for second tailnet") - reqbody := []byte(`{ - "keyType": "client", - "scopes": [ - "all" - ], - "tags": ["tag:k8s-operator"] - }`) - req, _ = http.NewRequestWithContext(ctx, "PUT", secondTSClient.BuildTailnetURL("keys", secondClientID), bytes.NewReader(reqbody)) - resp, err = secondTSClient.Do(req) + // Create an OAuth client for the second tailnet with the same + // scopes and tag as the primary tailnet's k8s-operator. + reqBody, err = json.Marshal(map[string]any{ + "keyType": "client", + "scopes": []string{"auth_keys", "devices:core", "services"}, + "tags": []string{"tag:k8s-operator"}, + "description": "k8s-operator client for e2e tests", + }) if err != nil { - return 0, fmt.Errorf("failed to set oauth scopes: %w", err) + return 0, fmt.Errorf("failed to marshal OAuth client creation request: %w", err) + } + req, _ = http.NewRequestWithContext(ctx, http.MethodPost, bootstrapClient.BuildTailnetURL("keys"), bytes.NewReader(reqBody)) + resp, err = bootstrapClient.Do(req) + if err != nil { + return 0, fmt.Errorf("failed to create OAuth client for second tailnet: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { b, _ := io.ReadAll(resp.Body) - return 0, fmt.Errorf("HTTP %d setting oauth scopes: %s", resp.StatusCode, string(b)) + return 0, fmt.Errorf("HTTP %d creating OAuth client for second tailnet: %s", resp.StatusCode, string(b)) } - logger.Infof("Oauth scopes configured") + var secondKey struct { + ID string `json:"id"` + Key string `json:"key"` + } + if err := json.NewDecoder(resp.Body).Decode(&secondKey); err != nil { + return 0, fmt.Errorf("failed to decode OAuth client creation response: %w", err) + } + secondClientID = secondKey.ID + secondClientSecret = secondKey.Key + logger.Info("set OAuth credentials for second tailnet") - // Set HTTPS on second Tailnet - req, _ = http.NewRequestWithContext(ctx, "PATCH", secondTSClient.BuildTailnetURL("settings"), bytes.NewBuffer([]byte(`{"httpsEnabled": true}`))) - resp, err = secondTSClient.Do(req) + secondTSClient, err = oauthTSClient(ctx, "http://localhost:31544", secondClientID, secondClientSecret) if err != nil { - return 0, fmt.Errorf("failed to enable HTTPS: %w", err) + return 0, fmt.Errorf("failed to set up second tailnet client: %w", err) } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return 0, fmt.Errorf("HTTP %d enabling https: %s", resp.StatusCode, string(b)) - } - logger.Infof("HTTPS settings configured") } else { clientSecret = os.Getenv("TS_API_CLIENT_SECRET") @@ -378,19 +388,10 @@ func runTests(m *testing.M) (int, error) { return 0, fmt.Errorf("TS_API_CLIENT_SECRET is not valid") } clientID = parts[2] - credentials := clientcredentials.Config{ - ClientID: clientID, - ClientSecret: clientSecret, - TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL), - Scopes: []string{"auth_keys"}, - } - tk, err := credentials.Token(ctx) + tsClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, clientID, clientSecret) if err != nil { - return 0, fmt.Errorf("failed to get OAuth token: %w", err) + return 0, fmt.Errorf("failed to set up primary tailnet client: %w", err) } - // An access token will last for an hour which is plenty of time for - // the tests to run. No need for token refresh logic. - tsClient = tailscale.NewClient("-", tailscale.APIKey(tk.AccessToken)) secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET") if secondClientSecret == "" { @@ -402,19 +403,10 @@ func runTests(m *testing.M) (int, error) { return 0, fmt.Errorf("SECOND_TS_API_CLIENT_SECRET is not valid") } secondClientID = parts[2] - secondCredentials := clientcredentials.Config{ - ClientID: secondClientID, - ClientSecret: secondClientSecret, - TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL), - Scopes: []string{"auth_keys"}, - } - secondTk, err := secondCredentials.Token(ctx) + secondTSClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, secondClientID, secondClientSecret) if err != nil { - return 0, fmt.Errorf("failed to get OAuth token: %w", err) + return 0, fmt.Errorf("failed to set up second tailnet client: %w", err) } - // An access token will last for an hour which is plenty of time for - // the tests to run. No need for token refresh logic. - secondTSClient = tailscale.NewClient("-", tailscale.APIKey(secondTk.AccessToken)) } var ossTag string @@ -886,22 +878,25 @@ func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts return nil } -func createTailnet(baseURL, apiKey string) (clientcredentials.Config, error) { +func createTailnet(ctx context.Context, cl *tailscale.Client) (clientID, clientSecret string, err error) { tailnetName := fmt.Sprintf("second-tailnet-%d", time.Now().Unix()) body, err := json.Marshal(map[string]any{"displayName": tailnetName}) if err != nil { - return clientcredentials.Config{}, err + return "", "", err } - req, _ := http.NewRequest("POST", baseURL+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body)) - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - resp, err := http.DefaultClient.Do(req) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + cl.BaseURL+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body)) if err != nil { - return clientcredentials.Config{}, err + return "", "", err + } + resp, err := cl.Do(req) + if err != nil { + return "", "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { b, _ := io.ReadAll(resp.Body) - return clientcredentials.Config{}, fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b)) + return "", "", fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b)) } var result struct { OauthClient struct { @@ -910,11 +905,25 @@ func createTailnet(baseURL, apiKey string) (clientcredentials.Config, error) { } `json:"oauthClient"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return clientcredentials.Config{}, fmt.Errorf("failed to decode response: %w", err) + return "", "", fmt.Errorf("failed to decode response: %w", err) } - return clientcredentials.Config{ - ClientID: result.OauthClient.ID, - ClientSecret: result.OauthClient.Secret, - TokenURL: baseURL + "/api/v2/oauth/token", - }, nil + return result.OauthClient.ID, result.OauthClient.Secret, nil +} + +// oauthTSClient exchanges OAuth client credentials for an access token and +// returns a tailscale.Client configured to use it. The token is valid for +// one hour, which is sufficient for a single test run. +func oauthTSClient(ctx context.Context, baseURL, clientID, clientSecret string) (*tailscale.Client, error) { + cfg := clientcredentials.Config{ + ClientID: clientID, + ClientSecret: clientSecret, + TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", baseURL), + } + tk, err := cfg.Token(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get OAuth token for client %q: %w", clientID, err) + } + c := tailscale.NewClient("-", tailscale.APIKey(tk.AccessToken)) + c.BaseURL = baseURL + return c, nil }