diff --git a/cmd/k8s-operator/e2e/multitailnet_test.go b/cmd/k8s-operator/e2e/multitailnet_test.go new file mode 100644 index 000000000..c19755129 --- /dev/null +++ b/cmd/k8s-operator/e2e/multitailnet_test.go @@ -0,0 +1,189 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package e2e + +import ( + "fmt" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstest" +) + +// TestMultiTailnet verifies that ProxyGroup resources are created in the correct Tailnet, +// and that an Ingress resource has its Tailscale Service created in the correct Tailnet. +func TestMultiTailnet(t *testing.T) { + if tnClient == nil || secondTSClient == nil { + t.Skip("TestMultiTailnet requires a working tailnet client for a primary and second tailnet") + } + + t.Log(secondClientID) + t.Log(secondClientSecret) + + // Create the tailnet Secret in the tailscale namespace. + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet-credentials", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_id": []byte(secondClientID), + "client_secret": []byte(secondClientSecret), + }, + } + createAndCleanup(t, kubeClient, secret) + + // Create the Tailnet resource. + tn := &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet", + }, + Spec: tsapi.TailnetSpec{ + LoginURL: clusterLoginServer, + Credentials: tsapi.TailnetCredentials{ + SecretName: "second-tailnet-credentials", + }, + }, + } + createAndCleanup(t, kubeClient, tn) + + // Apply nginx Deployment and Service. + createAndCleanup(t, kubeClient, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: new(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + }) + createAndCleanup(t, kubeClient, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + }) + + // Create Ingress ProxyGroup for each Tailnet. + firstTailnetPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first-tailnet", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + }, + } + createAndCleanup(t, kubeClient, firstTailnetPG) + secondTailnetPG := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeIngress, + Tailnet: "second-tailnet", + }, + } + createAndCleanup(t, kubeClient, secondTailnetPG) + + // TODO: Verify that devices have been created in the expected Tailnet. + + // Apply Ingress to expose nginx. + ingress := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-tailnet", + Namespace: "default", + Annotations: map[string]string{ + "tailscale.com/proxy-group": "second-tailnet", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: new("tailscale"), + TLS: []networkingv1.IngressTLS{ + networkingv1.IngressTLS{ + Hosts: []string{"second-tailnet"}, + }, + }, + Rules: []networkingv1.IngressRule{ + { + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: new(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "nginx", + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + createAndCleanup(t, kubeClient, ingress) + + if err := tstest.WaitFor(5*time.Minute, func() error { + ing := &networkingv1.Ingress{} + if err := kubeClient.Get(t.Context(), client.ObjectKey{ + Namespace: "default", Name: "second-tailnet", + }, ing); err != nil { + return err + } + if len(ing.Status.LoadBalancer.Ingress) == 0 || + ing.Status.LoadBalancer.Ingress[0].Hostname == "" { + return fmt.Errorf("Ingress not ready yet") + } + t.Logf("Ingress hostname: %s", ing.Status.LoadBalancer.Ingress[0].Hostname) + return nil + }); err != nil { + t.Fatalf("Ingress never got a hostname: %v", err) + } + + // TODO: cleanup second tailnet +} diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index 642ee57ec..a7cabb364 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -32,6 +32,7 @@ "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/tarball" "go.uber.org/zap" + "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -69,10 +70,14 @@ ) var ( - tsClient *tailscale.Client // For API calls to control. - tnClient *tsnet.Server // For testing real tailnet traffic. - restCfg *rest.Config // For constructing a client-go client if necessary. - kubeClient client.WithWatch // For k8s API calls. + tsClient *tailscale.Client // For API calls to control. + secondTSClient *tailscale.Client // For API calls to the secondary tailnet (_second_tailnet). + secondClientID string // OAuth client_id for second tailnet. + secondClientSecret string // OAuth client_secret for second tailnet. + tnClient *tsnet.Server // For testing real tailnet traffic. + restCfg *rest.Config // For constructing a client-go client if necessary. + kubeClient client.WithWatch // For k8s API calls. + clusterLoginServer string //go:embed certs/pebble.minica.crt pebbleMiniCACert []byte @@ -153,7 +158,6 @@ func runTests(m *testing.M) (int, error) { } var ( - clusterLoginServer string // Login server from cluster Pod point of view. clientID, clientSecret string // OAuth client for the operator to use. caPaths []string // Extra CA cert file paths to add to images. @@ -256,7 +260,7 @@ func runTests(m *testing.M) (int, error) { tsClient = tailscale.NewClient("-", tailscale.APIKey(apiKeyData.APIKey)) tsClient.BaseURL = "http://localhost:31544" - // Set ACLs and create OAuth client. + // Set ACLS and create Oauth client for primary tailnet req, _ := http.NewRequest("POST", tsClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs)) resp, err := tsClient.Do(req) if err != nil { @@ -268,7 +272,7 @@ func runTests(m *testing.M) (int, error) { return 0, fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b)) } logger.Infof("ACLs configured") - + logger.Info("set ACLs for primary tailnet") reqBody, err := json.Marshal(map[string]any{ "keyType": "client", "scopes": []string{"auth_keys", "devices:core", "services"}, @@ -297,7 +301,73 @@ func runTests(m *testing.M) (int, error) { } clientID = key.ID clientSecret = key.Key + logger.Info("set Oauth credentials for primary tailnet") + + // Create second tailnet. + secondClientCreds, err := createTailnet(tsClient.BaseURL, apiKeyData.APIKey) + if err != nil { + return 0, fmt.Errorf("failed to create second tailnet: %w", err) + } + + // Set up client for second tailnet- todo -get magic dns name from repsonse too? + secondClientID = secondClientCreds.ClientID + secondClientSecret = secondClientCreds.ClientSecret + source := secondClientCreds.TokenSource(ctx) + httpClient := oauth2.NewClient(ctx, source) + secondTSClient = tailscale.NewClient("-", nil) + secondTSClient.UserAgent = "e2e" + secondTSClient.HTTPClient = httpClient + secondTSClient.BaseURL = "http://localhost:31544" + + logger.Infof("OAUTH_CLIENT_ID=%s", secondClientID) + logger.Infof("OAUTH_CLIENT_SECRET=%s", secondClientSecret) + + // Set ACLs for second tailnet. + req, _ = http.NewRequest("POST", secondTSClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs)) + resp, err = secondTSClient.Do(req) + if err != nil { + return 0, fmt.Errorf("failed to set ACLs: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b)) + } + + reqbody := []byte(`{ + "keyType": "client", + "scopes": [ + "all" + ], + "tags": ["tag:k8s-operator"] + }`) + req, _ = http.NewRequest("PUT", secondTSClient.BuildTailnetURL("keys", secondClientID), bytes.NewReader(reqbody)) + resp, err = secondTSClient.Do(req) + if err != nil { + return 0, fmt.Errorf("failed to set oauth scopes: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("HTTP %d setting oauth scopes: %s", resp.StatusCode, string(b)) + } + logger.Infof("Oauth scopes configured") + + // Set HTTPS on second Tailnet + req, _ = http.NewRequest("PATCH", "http://localhost:31544/api/v2/tailnet/-/settings", bytes.NewBuffer([]byte(`{"httpsEnabled": true}`))) + resp, err = secondTSClient.Do(req) + if err != nil { + return 0, fmt.Errorf("failed to enable HTTPS: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("HTTP %d enabling https: %s", resp.StatusCode, string(b)) + } + logger.Infof("HTTPS settings configured") + } else { + // TODO: set up a tailscale client for the second tailnet clientSecret = os.Getenv("TS_API_CLIENT_SECRET") if clientSecret == "" { return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator") @@ -574,7 +644,6 @@ func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl c if err := cl.Patch(ctx, pc, client.Apply, owner); err != nil { return fmt.Errorf("failed to apply default ProxyClass: %w", err) } - // Wait for the ProxyClass to be marked ready. ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() @@ -727,3 +796,36 @@ func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts return nil } + +func createTailnet(baseURL, apiKey string) (clientcredentials.Config, error) { + tailnetName := fmt.Sprintf("second-tailnet-%d", time.Now().Unix()) + body, err := json.Marshal(map[string]any{"displayName": tailnetName}) + if err != nil { + return clientcredentials.Config{}, err + } + req, _ := http.NewRequest("POST", baseURL+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body)) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return clientcredentials.Config{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return clientcredentials.Config{}, fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b)) + } + var result struct { + OauthClient struct { + ID string `json:"id"` + Secret string `json:"secret"` + } `json:"oauthClient"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return clientcredentials.Config{}, fmt.Errorf("failed to decode response: %w", err) + } + return clientcredentials.Config{ + ClientID: result.OauthClient.ID, + ClientSecret: result.OauthClient.Secret, + TokenURL: baseURL + "/api/v2/oauth/token", + }, nil +} diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 28a836e97..5494d12af 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -138,16 +138,19 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque // to the Tailscale Service in a multi-cluster Ingress setup have not // resulted in another actor overwriting our Tailscale Service update. needsRequeue := false + shortRequeue := false if !ing.DeletionTimestamp.IsZero() || !r.shouldExpose(ing) { needsRequeue, err = r.maybeCleanup(ctx, hostname, ing, logger, tailscaleClient, pg) } else { - needsRequeue, err = r.maybeProvision(ctx, hostname, ing, logger, tailscaleClient, pg) + needsRequeue, shortRequeue, err = r.maybeProvision(ctx, hostname, ing, logger, tailscaleClient, pg) } if err != nil { return res, err } if needsRequeue { res = reconcile.Result{RequeueAfter: requeueInterval()} + } else if shortRequeue { + res = reconcile.Result{RequeueAfter: 30 * time.Second} } return res, nil } @@ -160,37 +163,37 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque // If a Tailscale Service exists, but does not have an owner reference from any operator, we error // out assuming that this is an owner reference created by an unknown actor. // Returns true if the operation resulted in a Tailscale Service update. -func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsClient, pg *tsapi.ProxyGroup) (svcsChanged bool, err error) { +func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsClient, pg *tsapi.ProxyGroup) (svcsChanged bool, shortRequeue bool, err error) { // Currently (2025-05) Tailscale Services are behind an alpha feature flag that // needs to be explicitly enabled for a tailnet to be able to use them. serviceName := tailcfg.ServiceName("svc:" + hostname) existingTSSvc, err := tsClient.GetVIPService(ctx, serviceName) if err != nil && !isErrorTailscaleServiceNotFound(err) { - return false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err) + return false, false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err) } if err = validateIngressClass(ctx, r.Client, r.ingressClassName); err != nil { logger.Infof("error validating tailscale IngressClass: %v.", err) - return false, nil + return false, false, nil } // Get and validate ProxyGroup readiness pgName := ing.Annotations[AnnotationProxyGroup] if pgName == "" { logger.Infof("[unexpected] no ProxyGroup annotation, skipping Tailscale Service provisioning") - return false, nil + return false, false, nil } logger = logger.With("ProxyGroup", pgName) if !tsoperator.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") - return false, nil + return false, false, nil } // Validate Ingress configuration if err := r.validateIngress(ctx, ing, pg); err != nil { logger.Infof("invalid Ingress configuration: %v", err) r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidIngressConfiguration", err.Error()) - return false, nil + return false, false, nil } if !IsHTTPSEnabledOnTailnet(r.tsnetServer) { @@ -205,7 +208,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin logger.Infof("exposing Ingress over tailscale") ing.Finalizers = append(ing.Finalizers, FinalizerNamePG) if err := r.Update(ctx, ing); err != nil { - return false, fmt.Errorf("failed to add finalizer: %w", err) + return false, false, fmt.Errorf("failed to add finalizer: %w", err) } r.mu.Lock() r.managedIngresses.Add(ing.UID) @@ -223,7 +226,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // (eventually) removed. svcsChanged, err = r.maybeCleanupProxyGroup(ctx, logger, tsClient, pg) if err != nil { - return false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err) + return false, false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err) } // 2. Ensure that there isn't a Tailscale Service with the same hostname @@ -243,31 +246,31 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) logger.Warn(msg) r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidTailscaleService", msg) - return false, nil + return false, false, nil } // 3. Ensure that TLS Secret and RBAC exists dnsName, err := dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace) if err != nil { - return false, fmt.Errorf("error determining DNS name for service: %w", err) + return false, false, fmt.Errorf("error determining DNS name for service: %w", err) } if err = r.ensureCertResources(ctx, pg, dnsName, ing); err != nil { - return false, fmt.Errorf("error ensuring cert resources: %w", err) + return false, false, fmt.Errorf("error ensuring cert resources: %w", err) } // 4. Ensure that the serve config for the ProxyGroup contains the Tailscale Service. cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName) if err != nil { - return false, fmt.Errorf("error getting Ingress serve config: %w", err) + return false, false, fmt.Errorf("error getting Ingress serve config: %w", err) } if cm == nil { logger.Infof("no Ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.") - return svcsChanged, nil + return svcsChanged, shortRequeue, nil } ep := ipn.HostPort(fmt.Sprintf("%s:443", dnsName)) handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, dnsName, logger) if err != nil { - return false, fmt.Errorf("failed to get handlers for Ingress: %w", err) + return false, false, fmt.Errorf("failed to get handlers for Ingress: %w", err) } ingCfg := &ipn.ServiceConfig{ TCP: map[uint16]*ipn.TCPPortHandler{ @@ -322,11 +325,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin mak.Set(&cfg.Services, serviceName, ingCfg) cfgBytes, err := json.Marshal(cfg) if err != nil { - return false, fmt.Errorf("error marshaling serve config: %w", err) + return false, false, fmt.Errorf("error marshaling serve config: %w", err) } mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes) if err := r.Update(ctx, cm); err != nil { - return false, fmt.Errorf("error updating serve config: %w", err) + return false, false, fmt.Errorf("error updating serve config: %w", err) } } @@ -360,7 +363,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin !ownersAreSetAndEqual(tsSvc, existingTSSvc) { logger.Infof("Ensuring Tailscale Service exists and is up to date") if err := tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil { - return false, fmt.Errorf("error creating Tailscale Service: %w", err) + return false, false, fmt.Errorf("error creating Tailscale Service: %w", err) } } @@ -370,14 +373,19 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin if isHTTPEndpointEnabled(ing) || isHTTPRedirectEnabled(ing) { mode = serviceAdvertisementHTTPAndHTTPS } - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg); err != nil { - return false, fmt.Errorf("failed to update tailscaled config: %w", err) + shouldBeAdvertised, err := r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg) + if err != nil { + return false, false, fmt.Errorf("failed to update tailscaled config: %w", err) + } + // If certs are not yet ready, schedule a short requeue. + if !shouldBeAdvertised { + shortRequeue = true } // 6. Update Ingress status if ProxyGroup Pods are ready. count, err := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName) if err != nil { - return false, fmt.Errorf("failed to check if any Pods are configured: %w", err) + return false, false, fmt.Errorf("failed to check if any Pods are configured: %w", err) } oldStatus := ing.Status.DeepCopy() @@ -385,11 +393,14 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin switch count { case 0: ing.Status.LoadBalancer.Ingress = nil + if shouldBeAdvertised { + shortRequeue = true + } default: var ports []networkingv1.IngressPortStatus hasCerts, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) if err != nil { - return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) + return false, false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) } // If TLS certs have not been issued (yet), do not set port 443. if hasCerts { @@ -417,7 +428,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } } if apiequality.Semantic.DeepEqual(oldStatus, &ing.Status) { - return svcsChanged, nil + return svcsChanged, shortRequeue, nil } const prefix = "Updating Ingress status" @@ -428,10 +439,10 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } if err = r.Status().Update(ctx, ing); err != nil { - return false, fmt.Errorf("failed to update Ingress status: %w", err) + return false, false, fmt.Errorf("failed to update Ingress status: %w", err) } - return svcsChanged, nil + return svcsChanged, shortRequeue, nil } // maybeCleanupProxyGroup ensures that any Tailscale Services that are @@ -485,7 +496,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger } // Make sure the Tailscale Service is not advertised in tailscaled or serve config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil { + if _, err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } @@ -571,7 +582,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, } // 4. Unadvertise the Tailscale Service in tailscaled config. - if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil { + if _, err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } @@ -755,11 +766,11 @@ func isHTTPEndpointEnabled(ing *networkingv1.Ingress) bool { serviceAdvertisementHTTPAndHTTPS // Both ports 80 and 443 should be advertised ) -func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup) (err error) { +func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup) (shouldBeAdvertised bool, err error) { // Get all config Secrets for this ProxyGroup. secrets := &corev1.SecretList{} if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil { - return fmt.Errorf("failed to list config Secrets: %w", err) + return false, fmt.Errorf("failed to list config Secrets: %w", err) } // Verify that TLS cert for the Tailscale Service has been successfully issued @@ -772,9 +783,9 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // TLS cert is not yet provisioned. hasCert, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg) if err != nil { - return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) + return false, fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) } - shouldBeAdvertised := (mode == serviceAdvertisementHTTPAndHTTPS) || + shouldBeAdvertised = (mode == serviceAdvertisementHTTPAndHTTPS) || (mode == serviceAdvertisementHTTPS && hasCert) // if we only expose port 443 and don't have certs (yet), do not advertise for _, secret := range secrets.Items { @@ -782,7 +793,7 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con for fileName, confB := range secret.Data { var conf ipn.ConfigVAlpha if err := json.Unmarshal(confB, &conf); err != nil { - return fmt.Errorf("error unmarshalling ProxyGroup config: %w", err) + return false, fmt.Errorf("error unmarshalling ProxyGroup config: %w", err) } // Update the services to advertise if required. @@ -803,7 +814,7 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // Update the Secret. confB, err := json.Marshal(conf) if err != nil { - return fmt.Errorf("error marshalling ProxyGroup config: %w", err) + return false, fmt.Errorf("error marshalling ProxyGroup config: %w", err) } mak.Set(&secret.Data, fileName, confB) updated = true @@ -811,12 +822,12 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con if updated { if err := r.Update(ctx, &secret); err != nil { - return fmt.Errorf("error updating ProxyGroup config Secret: %w", err) + return false, fmt.Errorf("error updating ProxyGroup config Secret: %w", err) } } } - return nil + return shouldBeAdvertised, nil } func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, pgName string, serviceName tailcfg.ServiceName) (int, error) {