diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go index 7cf05dd8d..3bcd464aa 100644 --- a/cmd/k8s-operator/e2e/ingress_test.go +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -5,7 +5,6 @@ import ( "context" - "crypto/tls" "fmt" "net/http" "strings" @@ -21,7 +20,6 @@ kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" - "tailscale.com/tsnet" "tailscale.com/tstest" "tailscale.com/util/httpm" ) @@ -276,6 +274,89 @@ func TestL7HAIngress(t *testing.T) { } } +// TestMultiTailnet verifies that ProxyGroup resources are created in the correct Tailnet, +// and that an Ingress resource has its Tailscale Service created in the correct Tailnet. +func TestL7HAMultiTailnet(t *testing.T) { + if tnClient == nil || secondTSClient == nil { + t.Skip("TestMultiTailnet requires a working tailnet client for a primary and second tailnet") + } + + // Apply nginx Deployment and Service. + createAndCleanup(t, kubeClient, nginxDeployment("default", "nginx")) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + // Create Ingress ProxyGroup for each Tailnet. + firstTailnetPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first-tailnet", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + } + createAndCleanup(t, kubeClient, firstTailnetPG) + secondTailnetPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + Tailnet: "second-tailnet", + }, + } + createAndCleanup(t, kubeClient, secondTailnetPG) + + // TODO: Verify that devices have been created in the expected Tailnet. + + // Apply Ingress to expose nginx. + name := generateName("ingress") + ingress := l7Ingress(ns, name, map[string]string{ + "tailscale.com/proxy-group": "second-tailnet", + }) + createAndCleanup(t, kubeClient, ingress) + + hostname, err := waitForIngressHostname(t, ns, name) + if err != nil { + t.Fatalf("error waiting for Ingress hostname: %v", err) + } + httpClient := newHTTPClient(secondTNClient) + + var resp *http.Response + if err := tstest.WaitFor(time.Minute, func() error { + req, err := http.NewRequest(httpm.GET, fmt.Sprintf("https://%s:443", hostname), nil) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + resp, err = httpClient.Do(req.WithContext(ctx)) + return err + }); err != nil { + t.Fatalf("error trying to reach Ingress: %v", err) + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status: %v", resp.StatusCode) + } +} + func l7Ingress(namespace, name string, annotations map[string]string) *networkingv1.Ingress { ingress := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -315,43 +396,6 @@ func l7Ingress(namespace, name string, annotations map[string]string) *networkin return ingress } -// waitForIngressHostname polls until the named Ingress has a LoadBalancer -// hostname assigned, then returns that hostname. -func waitForIngressHostname(t *testing.T, namespace, name string) (string, error) { - t.Helper() - var hostname string - if err := tstest.WaitFor(3*time.Minute, func() error { - ing := &networkingv1.Ingress{} - if err := kubeClient.Get(t.Context(), client.ObjectKey{ - Namespace: namespace, Name: name, - }, ing); err != nil { - return err - } - if len(ing.Status.LoadBalancer.Ingress) == 0 || - ing.Status.LoadBalancer.Ingress[0].Hostname == "" { - return fmt.Errorf("Ingress not ready yet") - } - t.Logf("Ingress hostname: %s", ing.Status.LoadBalancer.Ingress[0].Hostname) - hostname = ing.Status.LoadBalancer.Ingress[0].Hostname - return nil - }); err != nil { - return "", fmt.Errorf("Ingress %s/%s never got a hostname: %w", namespace, name, err) - } - return hostname, nil -} - -// tailnetHTTPSClient returns an HTTP client that routes connections through -// the given tsnet.Server and trusts the test CA pool. -func tailnetHTTPSClient(srv *tsnet.Server) *http.Client { - return &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: testCAs}, - DialContext: srv.Dial, - }, - } -} - -// nginxDeployment returns a simple nginx Deployment for use in tests. func nginxDeployment(namespace, name string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -428,7 +472,7 @@ func testIngressIsReachable(t *testing.T, httpClient *http.Client, url string) e if err != nil { return err } - ctx, cancel := context.WithTimeout(t.Context(), time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() resp, err = httpClient.Do(req.WithContext(ctx)) if err != nil { @@ -439,7 +483,6 @@ func testIngressIsReachable(t *testing.T, httpClient *http.Client, url string) e }); err != nil { return fmt.Errorf("error trying to reach %s: %w", url, err) } - if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status from %s: %d", url, resp.StatusCode) } diff --git a/cmd/k8s-operator/e2e/multitailnet_test.go b/cmd/k8s-operator/e2e/multitailnet_test.go deleted file mode 100644 index 7669af871..000000000 --- a/cmd/k8s-operator/e2e/multitailnet_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -package e2e - -import ( - "context" - "fmt" - "net/http" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - tsapi "tailscale.com/k8s-operator/apis/v1alpha1" - "tailscale.com/tstest" - "tailscale.com/util/httpm" -) - -// TestMultiTailnet verifies that ProxyGroup resources are created in the correct Tailnet, -// and that an Ingress resource has its Tailscale Service created in the correct Tailnet. -func TestMultiTailnet(t *testing.T) { - if tnClient == nil || secondTSClient == nil { - t.Skip("TestMultiTailnet requires a working tailnet client for a primary and second tailnet") - } - - // Apply nginx Deployment and Service. - createAndCleanup(t, kubeClient, nginxDeployment("default", "nginx")) - createAndCleanup(t, kubeClient, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nginx", - Namespace: "default", - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "app.kubernetes.io/name": "nginx", - }, - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: 80, - }, - }, - }, - }) - - // Create Ingress ProxyGroup for each Tailnet. - firstTailnetPG := &tsapi.ProxyGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "first-tailnet", - }, - Spec: tsapi.ProxyGroupSpec{ - Type: tsapi.ProxyGroupTypeIngress, - }, - } - createAndCleanup(t, kubeClient, firstTailnetPG) - secondTailnetPG := &tsapi.ProxyGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "second-tailnet", - }, - Spec: tsapi.ProxyGroupSpec{ - Type: tsapi.ProxyGroupTypeIngress, - Tailnet: "second-tailnet", - }, - } - createAndCleanup(t, kubeClient, secondTailnetPG) - - // TODO: Verify that devices have been created in the expected Tailnet. - - // Apply Ingress to expose nginx. - ingress := &networkingv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "second-tailnet", - Namespace: "default", - Annotations: map[string]string{ - "tailscale.com/proxy-group": "second-tailnet", - }, - }, - Spec: networkingv1.IngressSpec{ - IngressClassName: new("tailscale"), - TLS: []networkingv1.IngressTLS{ - networkingv1.IngressTLS{ - Hosts: []string{"second-tailnet"}, - }, - }, - Rules: []networkingv1.IngressRule{ - { - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - { - Path: "/", - PathType: new(networkingv1.PathTypePrefix), - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: "nginx", - Port: networkingv1.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - createAndCleanup(t, kubeClient, ingress) - - hostname, err := waitForIngressHostname(t, "default", "second-tailnet") - if err != nil { - t.Fatalf("error waiting for Ingress hostname: %v", err) - } - httpClient := tailnetHTTPSClient(secondTNClient) - - var resp *http.Response - if err := tstest.WaitFor(time.Minute, func() error { - req, err := http.NewRequest(httpm.GET, fmt.Sprintf("https://%s:443", hostname), nil) - if err != nil { - return err - } - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - defer cancel() - resp, err = httpClient.Do(req.WithContext(ctx)) - return err - }); err != nil { - t.Fatalf("error trying to reach Ingress: %v", err) - } - - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status: %v", resp.StatusCode) - } -} diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index 2c4856ade..d0b3e57c8 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -263,116 +263,45 @@ func runTests(m *testing.M) (int, error) { tsClient.BaseURL = "http://localhost:31544" // Set ACLS and create Oauth client for primary tailnet - req, _ := http.NewRequestWithContext(ctx, "POST", tsClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs)) - resp, err := tsClient.Do(req) + err = setACLs(ctx, tsClient) if err != nil { - return 0, fmt.Errorf("failed to set ACLs: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return 0, fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b)) + return 0, err } logger.Info("set ACLs for primary tailnet") - reqBody, err := json.Marshal(map[string]any{ - "keyType": "client", - "scopes": []string{"auth_keys", "devices:core", "services"}, - "tags": []string{"tag:k8s-operator"}, - "description": "k8s-operator client for e2e tests", - }) - if err != nil { - return 0, fmt.Errorf("failed to marshal OAuth client creation request: %w", err) - } - req, _ = http.NewRequestWithContext(ctx, "POST", tsClient.BuildTailnetURL("keys"), bytes.NewReader(reqBody)) - resp, err = tsClient.Do(req) - if err != nil { - return 0, fmt.Errorf("failed to create OAuth client: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return 0, fmt.Errorf("HTTP %d creating OAuth client: %s", resp.StatusCode, string(b)) - } - var key struct { - ID string `json:"id"` - Key string `json:"key"` - } - if err := json.NewDecoder(resp.Body).Decode(&key); err != nil { - return 0, fmt.Errorf("failed to decode OAuth client creation response: %w", err) - } - clientID = key.ID - clientSecret = key.Key + clientID, clientSecret, err = createOauthCreds(ctx, tsClient) logger.Info("set Oauth credentials for primary tailnet") // Create second tailnet. The bootstrap credentials returned have all permissions- - // they are used only to create an OAuth client for the k8s-operator - bootstrapID, bootstrapSecret, err := createTailnet(ctx, tsClient) + // they are used only to create an OAuth client for the k8s-operator. + bootstrapSecret, err := createTailnet(ctx, tsClient) if err != nil { return 0, fmt.Errorf("failed to create second tailnet: %w", err) } - bootstrapClient, err := oauthTSClient(ctx, "http://localhost:31544", bootstrapID, bootstrapSecret) + bootstrapClient, err := oauthTSClient(ctx, "http://localhost:31544", bootstrapSecret) if err != nil { return 0, fmt.Errorf("failed to set up bootstrap client for second tailnet: %w", err) } + // Set HTTPS on second tailnet. - req, _ = http.NewRequestWithContext(ctx, http.MethodPatch, bootstrapClient.BuildTailnetURL("settings"), - bytes.NewBufferString(`{"httpsEnabled": true}`)) - resp, err = bootstrapClient.Do(req) + err = setHTTPSForTailnet(ctx, bootstrapClient) if err != nil { - return 0, fmt.Errorf("failed to enable HTTPS: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return 0, fmt.Errorf("HTTP %d enabling HTTPS: %s", resp.StatusCode, string(b)) + return 0, err } logger.Info("HTTPS settings configured for second tailnet") + // Set ACLs for second tailnet. - req, _ = http.NewRequestWithContext(ctx, http.MethodPost, bootstrapClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs)) - resp, err = bootstrapClient.Do(req) + err = setACLs(ctx, bootstrapClient) if err != nil { - return 0, fmt.Errorf("failed to set ACLs: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return 0, fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b)) + return 0, err } logger.Info("set ACLs for second tailnet") // Create an OAuth client for the second tailnet with the same // scopes and tag as the primary tailnet's k8s-operator. - reqBody, err = json.Marshal(map[string]any{ - "keyType": "client", - "scopes": []string{"auth_keys", "devices:core", "services"}, - "tags": []string{"tag:k8s-operator"}, - "description": "k8s-operator client for e2e tests", - }) - if err != nil { - return 0, fmt.Errorf("failed to marshal OAuth client creation request: %w", err) - } - req, _ = http.NewRequestWithContext(ctx, http.MethodPost, bootstrapClient.BuildTailnetURL("keys"), bytes.NewReader(reqBody)) - resp, err = bootstrapClient.Do(req) - if err != nil { - return 0, fmt.Errorf("failed to create OAuth client for second tailnet: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return 0, fmt.Errorf("HTTP %d creating OAuth client for second tailnet: %s", resp.StatusCode, string(b)) - } - var secondKey struct { - ID string `json:"id"` - Key string `json:"key"` - } - if err := json.NewDecoder(resp.Body).Decode(&secondKey); err != nil { - return 0, fmt.Errorf("failed to decode OAuth client creation response: %w", err) - } - secondClientID = secondKey.ID - secondClientSecret = secondKey.Key + secondClientID, secondClientSecret, err = createOauthCreds(ctx, bootstrapClient) logger.Info("set OAuth credentials for second tailnet") - secondTSClient, err = oauthTSClient(ctx, "http://localhost:31544", secondClientID, secondClientSecret) + secondTSClient, err = oauthTSClient(ctx, "http://localhost:31544", secondClientSecret) if err != nil { return 0, fmt.Errorf("failed to set up second tailnet client: %w", err) } @@ -382,31 +311,30 @@ func runTests(m *testing.M) (int, error) { if clientSecret == "" { return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") } - // Format is "tskey-client--". + // TODO: remove duplication parts := strings.Split(clientSecret, "-") if len(parts) != 4 { return 0, fmt.Errorf("TS_API_CLIENT_SECRET is not valid") } clientID = parts[2] - tsClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, clientID, clientSecret) + tsClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, clientSecret) if err != nil { return 0, fmt.Errorf("failed to set up primary tailnet client: %w", err) } - secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET") if secondClientSecret == "" { return 0, fmt.Errorf("must use --devcontrol or set SECOND_TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") } - // Format is "tskey-client--". parts = strings.Split(secondClientSecret, "-") if len(parts) != 4 { return 0, fmt.Errorf("SECOND_TS_API_CLIENT_SECRET is not valid") } secondClientID = parts[2] - secondTSClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, secondClientID, secondClientSecret) + secondTSClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, secondClientSecret) if err != nil { return 0, fmt.Errorf("failed to set up second tailnet client: %w", err) } + } var ossTag string @@ -621,6 +549,65 @@ func runTests(m *testing.M) (int, error) { return m.Run(), nil } +func setHTTPSForTailnet(ctx context.Context, client *tailscale.Client) error { + req, _ := http.NewRequestWithContext(ctx, http.MethodPatch, client.BuildTailnetURL("settings"), + bytes.NewBufferString(`{"httpsEnabled": true}`)) + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to enable HTTPS: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d enabling HTTPS: %s", resp.StatusCode, string(b)) + } + return nil +} + +func setACLs(ctx context.Context, client *tailscale.Client) error { + req, _ := http.NewRequestWithContext(ctx, "POST", tsClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs)) + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to set ACLs: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b)) + } + return nil +} + +func createOauthCreds(ctx context.Context, client *tailscale.Client) (string, string, error) { + reqBody, err := json.Marshal(map[string]any{ + "keyType": "client", + "scopes": []string{"auth_keys", "devices:core", "services"}, + "tags": []string{"tag:k8s-operator"}, + "description": "k8s-operator client for e2e tests", + }) + if err != nil { + return "", "", fmt.Errorf("failed to marshal OAuth client creation request: %w", err) + } + req, _ := http.NewRequestWithContext(ctx, "POST", client.BuildTailnetURL("keys"), bytes.NewReader(reqBody)) + resp, err := tsClient.Do(req) + if err != nil { + return "", "", fmt.Errorf("failed to create OAuth client: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return "", "", fmt.Errorf("HTTP %d creating OAuth client: %s", resp.StatusCode, string(b)) + } + var key struct { + ID string `json:"id"` + Key string `json:"key"` + } + if err := json.NewDecoder(resp.Body).Decode(&key); err != nil { + return "", "", fmt.Errorf("failed to decode OAuth client creation response: %w", err) + } + return key.ID, key.Key, nil +} + func upgraderOrInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc { hist := action.NewHistory(cfg) hist.Max = 1 @@ -878,25 +865,25 @@ func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts return nil } -func createTailnet(ctx context.Context, cl *tailscale.Client) (clientID, clientSecret string, err error) { +func createTailnet(ctx context.Context, cl *tailscale.Client) (clientSecret string, err error) { tailnetName := fmt.Sprintf("second-tailnet-%d", time.Now().Unix()) body, err := json.Marshal(map[string]any{"displayName": tailnetName}) if err != nil { - return "", "", err + return "", err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, cl.BaseURL+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body)) if err != nil { - return "", "", err + return "", err } resp, err := cl.Do(req) if err != nil { - return "", "", err + return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { b, _ := io.ReadAll(resp.Body) - return "", "", fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b)) + return "", fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b)) } var result struct { OauthClient struct { @@ -905,15 +892,20 @@ func createTailnet(ctx context.Context, cl *tailscale.Client) (clientID, clientS } `json:"oauthClient"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return "", "", fmt.Errorf("failed to decode response: %w", err) + return "", fmt.Errorf("failed to decode response: %w", err) } - return result.OauthClient.ID, result.OauthClient.Secret, nil + return result.OauthClient.Secret, nil } // oauthTSClient exchanges OAuth client credentials for an access token and // returns a tailscale.Client configured to use it. The token is valid for // one hour, which is sufficient for a single test run. -func oauthTSClient(ctx context.Context, baseURL, clientID, clientSecret string) (*tailscale.Client, error) { +func oauthTSClient(ctx context.Context, baseURL, clientSecret string) (*tailscale.Client, error) { + parts := strings.Split(clientSecret, "-") + if len(parts) != 4 { + return nil, fmt.Errorf("TS_API_CLIENT_SECRET is not valid") + } + clientID := parts[2] cfg := clientcredentials.Config{ ClientID: clientID, ClientSecret: clientSecret, diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 5494d12af..28a836e97 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -138,19 +138,16 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque // to the Tailscale Service in a multi-cluster Ingress setup have not // resulted in another actor overwriting our Tailscale Service update. needsRequeue := false - shortRequeue := false if !ing.DeletionTimestamp.IsZero() || !r.shouldExpose(ing) { needsRequeue, err = r.maybeCleanup(ctx, hostname, ing, logger, tailscaleClient, pg) } else { - needsRequeue, shortRequeue, err = r.maybeProvision(ctx, hostname, ing, logger, tailscaleClient, pg) + needsRequeue, err = r.maybeProvision(ctx, hostname, ing, logger, tailscaleClient, pg) } if err != nil { return res, err } if needsRequeue { res = reconcile.Result{RequeueAfter: requeueInterval()} - } else if shortRequeue { - res = reconcile.Result{RequeueAfter: 30 * time.Second} } return res, nil } @@ -163,37 +160,37 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque // If a Tailscale Service exists, but does not have an owner reference from any operator, we error // out assuming that this is an owner reference created by an unknown actor. // Returns true if the operation resulted in a Tailscale Service update. -func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsClient, pg *tsapi.ProxyGroup) (svcsChanged bool, shortRequeue bool, err error) { +func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsClient, pg *tsapi.ProxyGroup) (svcsChanged bool, err error) { // Currently (2025-05) Tailscale Services are behind an alpha feature flag that // needs to be explicitly enabled for a tailnet to be able to use them. serviceName := tailcfg.ServiceName("svc:" + hostname) existingTSSvc, err := tsClient.GetVIPService(ctx, serviceName) if err != nil && !isErrorTailscaleServiceNotFound(err) { - return false, false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err) + return false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err) } if err = validateIngressClass(ctx, r.Client, r.ingressClassName); err != nil { logger.Infof("error validating tailscale IngressClass: %v.", err) - return false, false, nil + return false, nil } // Get and validate ProxyGroup readiness pgName := ing.Annotations[AnnotationProxyGroup] if pgName == "" { logger.Infof("[unexpected] no ProxyGroup annotation, skipping Tailscale Service provisioning") - return false, false, nil + return false, nil } logger = logger.With("ProxyGroup", pgName) if !tsoperator.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") - return false, false, nil + return false, nil } // Validate Ingress configuration if err := r.validateIngress(ctx, ing, pg); err != nil { logger.Infof("invalid Ingress configuration: %v", err) r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidIngressConfiguration", err.Error()) - return false, false, nil + return false, nil } if !IsHTTPSEnabledOnTailnet(r.tsnetServer) { @@ -208,7 +205,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin logger.Infof("exposing Ingress over tailscale") ing.Finalizers = append(ing.Finalizers, FinalizerNamePG) if err := r.Update(ctx, ing); err != nil { - return false, false, fmt.Errorf("failed to add finalizer: %w", err) + return false, fmt.Errorf("failed to add finalizer: %w", err) } r.mu.Lock() r.managedIngresses.Add(ing.UID) @@ -226,7 +223,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // (eventually) removed. svcsChanged, err = r.maybeCleanupProxyGroup(ctx, logger, tsClient, pg) if err != nil { - return false, false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err) + return false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err) } // 2. Ensure that there isn't a Tailscale Service with the same hostname @@ -246,31 +243,31 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) logger.Warn(msg) r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidTailscaleService", msg) - return false, false, nil + return false, nil } // 3. Ensure that TLS Secret and RBAC exists dnsName, err := dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace) if err != nil { - return false, false, fmt.Errorf("error determining DNS name for service: %w", err) + return false, fmt.Errorf("error determining DNS name for service: %w", err) } if err = r.ensureCertResources(ctx, pg, dnsName, ing); err != nil { - return false, false, fmt.Errorf("error ensuring cert resources: %w", err) + return false, fmt.Errorf("error ensuring cert resources: %w", err) } // 4. Ensure that the serve config for the ProxyGroup contains the Tailscale Service. cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName) if err != nil { - return false, false, fmt.Errorf("error getting Ingress serve config: %w", err) + return false, fmt.Errorf("error getting Ingress serve config: %w", err) } if cm == nil { logger.Infof("no Ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.") - return svcsChanged, shortRequeue, nil + return svcsChanged, nil } ep := ipn.HostPort(fmt.Sprintf("%s:443", dnsName)) handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, dnsName, logger) if err != nil { - return false, false, fmt.Errorf("failed to get handlers for Ingress: %w", err) + return false, fmt.Errorf("failed to get handlers for Ingress: %w", err) } ingCfg := &ipn.ServiceConfig{ TCP: map[uint16]*ipn.TCPPortHandler{ @@ -325,11 +322,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin mak.Set(&cfg.Services, serviceName, ingCfg) cfgBytes, err := json.Marshal(cfg) if err != nil { - return false, false, fmt.Errorf("error marshaling serve config: %w", err) + return false, fmt.Errorf("error marshaling serve config: %w", err) } mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes) if err := r.Update(ctx, cm); err != nil { - return false, false, fmt.Errorf("error updating serve config: %w", err) + return false, fmt.Errorf("error updating serve config: %w", err) } } @@ -363,7 +360,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin !ownersAreSetAndEqual(tsSvc, existingTSSvc) { logger.Infof("Ensuring Tailscale Service exists and is up to date") if err := tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil { - return false, false, fmt.Errorf("error creating Tailscale Service: %w", err) + return false, fmt.Errorf("error creating Tailscale Service: %w", err) } } @@ -373,19 +370,14 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin if isHTTPEndpointEnabled(ing) || isHTTPRedirectEnabled(ing) { mode = serviceAdvertisementHTTPAndHTTPS } - shouldBeAdvertised, err := r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg) - if err != nil { - return false, false, fmt.Errorf("failed to update tailscaled config: %w", err) - } - // If certs are not yet ready, schedule a short requeue. - if !shouldBeAdvertised { - shortRequeue = true + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg); err != nil { + return false, fmt.Errorf("failed to update tailscaled config: %w", err) } // 6. Update Ingress status if ProxyGroup Pods are ready. count, err := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName) if err != nil { - return false, false, fmt.Errorf("failed to check if any Pods are configured: %w", err) + return false, fmt.Errorf("failed to check if any Pods are configured: %w", err) } oldStatus := ing.Status.DeepCopy() @@ -393,14 +385,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin switch count { case 0: ing.Status.LoadBalancer.Ingress = nil - if shouldBeAdvertised { - shortRequeue = true - } default: var ports []networkingv1.IngressPortStatus hasCerts, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) if err != nil { - return false, false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) + return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) } // If TLS certs have not been issued (yet), do not set port 443. if hasCerts { @@ -428,7 +417,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } } if apiequality.Semantic.DeepEqual(oldStatus, &ing.Status) { - return svcsChanged, shortRequeue, nil + return svcsChanged, nil } const prefix = "Updating Ingress status" @@ -439,10 +428,10 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } if err = r.Status().Update(ctx, ing); err != nil { - return false, false, fmt.Errorf("failed to update Ingress status: %w", err) + return false, fmt.Errorf("failed to update Ingress status: %w", err) } - return svcsChanged, shortRequeue, nil + return svcsChanged, nil } // maybeCleanupProxyGroup ensures that any Tailscale Services that are @@ -496,7 +485,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger } // Make sure the Tailscale Service is not advertised in tailscaled or serve config. - if _, err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } @@ -582,7 +571,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, } // 4. Unadvertise the Tailscale Service in tailscaled config. - if _, err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil { + if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } @@ -766,11 +755,11 @@ func isHTTPEndpointEnabled(ing *networkingv1.Ingress) bool { serviceAdvertisementHTTPAndHTTPS // Both ports 80 and 443 should be advertised ) -func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup) (shouldBeAdvertised bool, err error) { +func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup) (err error) { // Get all config Secrets for this ProxyGroup. secrets := &corev1.SecretList{} if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil { - return false, fmt.Errorf("failed to list config Secrets: %w", err) + return fmt.Errorf("failed to list config Secrets: %w", err) } // Verify that TLS cert for the Tailscale Service has been successfully issued @@ -783,9 +772,9 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // TLS cert is not yet provisioned. hasCert, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) if err != nil { - return false, fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) + return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) } - shouldBeAdvertised = (mode == serviceAdvertisementHTTPAndHTTPS) || + shouldBeAdvertised := (mode == serviceAdvertisementHTTPAndHTTPS) || (mode == serviceAdvertisementHTTPS && hasCert) // if we only expose port 443 and don't have certs (yet), do not advertise for _, secret := range secrets.Items { @@ -793,7 +782,7 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con for fileName, confB := range secret.Data { var conf ipn.ConfigVAlpha if err := json.Unmarshal(confB, &conf); err != nil { - return false, fmt.Errorf("error unmarshalling ProxyGroup config: %w", err) + return fmt.Errorf("error unmarshalling ProxyGroup config: %w", err) } // Update the services to advertise if required. @@ -814,7 +803,7 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // Update the Secret. confB, err := json.Marshal(conf) if err != nil { - return false, fmt.Errorf("error marshalling ProxyGroup config: %w", err) + return fmt.Errorf("error marshalling ProxyGroup config: %w", err) } mak.Set(&secret.Data, fileName, confB) updated = true @@ -822,12 +811,12 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con if updated { if err := r.Update(ctx, &secret); err != nil { - return false, fmt.Errorf("error updating ProxyGroup config Secret: %w", err) + return fmt.Errorf("error updating ProxyGroup config Secret: %w", err) } } } - return shouldBeAdvertised, nil + return nil } func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, pgName string, serviceName tailcfg.ServiceName) (int, error) {