mirror/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2026-04-03 06:02:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

move test, refactor setup

Browse Source
This commit is contained in:
Becky Pauley
2026-04-02 20:12:10 +01:00
parent a889f4a4f1
commit 71be6aeb6d
4 changed files with 209 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
cmd/k8s-operator/e2e/ingress_test.go
View File

@@ -5,7 +5,6 @@
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"strings"
@@ -21,7 +20,6 @@
kube "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
"tailscale.com/tsnet"
"tailscale.com/tstest"
"tailscale.com/util/httpm"
)
@@ -276,6 +274,89 @@ func TestL7HAIngress(t *testing.T) {
}
}
// TestMultiTailnet verifies that ProxyGroup resources are created in the correct Tailnet,
// and that an Ingress resource has its Tailscale Service created in the correct Tailnet.
func TestL7HAMultiTailnet(t *testing.T) {
if tnClient == nil || secondTSClient == nil {
t.Skip("TestMultiTailnet requires a working tailnet client for a primary and second tailnet")
}
// Apply nginx Deployment and Service.
createAndCleanup(t, kubeClient, nginxDeployment("default", "nginx"))
createAndCleanup(t, kubeClient, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "nginx",
Namespace: "default",
},
Spec: corev1.ServiceSpec{
Selector: map[string]string{
"app.kubernetes.io/name": "nginx",
},
Ports: []corev1.ServicePort{
{
Name: "http",
Port: 80,
},
},
},
})
// Create Ingress ProxyGroup for each Tailnet.
firstTailnetPG := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "first-tailnet",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
},
}
createAndCleanup(t, kubeClient, firstTailnetPG)
secondTailnetPG := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "second-tailnet",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
Tailnet: "second-tailnet",
},
}
createAndCleanup(t, kubeClient, secondTailnetPG)
// TODO: Verify that devices have been created in the expected Tailnet.
// Apply Ingress to expose nginx.
name := generateName("ingress")
ingress := l7Ingress(ns, name, map[string]string{
"tailscale.com/proxy-group": "second-tailnet",
})
createAndCleanup(t, kubeClient, ingress)
hostname, err := waitForIngressHostname(t, ns, name)
if err != nil {
t.Fatalf("error waiting for Ingress hostname: %v", err)
}
httpClient := newHTTPClient(secondTNClient)
var resp *http.Response
if err := tstest.WaitFor(time.Minute, func() error {
req, err := http.NewRequest(httpm.GET, fmt.Sprintf("https://%s:443", hostname), nil)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
resp, err = httpClient.Do(req.WithContext(ctx))
return err
}); err != nil {
t.Fatalf("error trying to reach Ingress: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %v", resp.StatusCode)
}
}
func l7Ingress(namespace, name string, annotations map[string]string) *networkingv1.Ingress {
ingress := &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
@@ -315,43 +396,6 @@ func l7Ingress(namespace, name string, annotations map[string]string) *networkin
return ingress
}
// waitForIngressHostname polls until the named Ingress has a LoadBalancer
// hostname assigned, then returns that hostname.
func waitForIngressHostname(t *testing.T, namespace, name string) (string, error) {
t.Helper()
var hostname string
if err := tstest.WaitFor(3*time.Minute, func() error {
ing := &networkingv1.Ingress{}
if err := kubeClient.Get(t.Context(), client.ObjectKey{
Namespace: namespace, Name: name,
}, ing); err != nil {
return err
}
if len(ing.Status.LoadBalancer.Ingress) == 0 ||
ing.Status.LoadBalancer.Ingress[0].Hostname == "" {
return fmt.Errorf("Ingress not ready yet")
}
t.Logf("Ingress hostname: %s", ing.Status.LoadBalancer.Ingress[0].Hostname)
hostname = ing.Status.LoadBalancer.Ingress[0].Hostname
return nil
}); err != nil {
return "", fmt.Errorf("Ingress %s/%s never got a hostname: %w", namespace, name, err)
}
return hostname, nil
}
// tailnetHTTPSClient returns an HTTP client that routes connections through
// the given tsnet.Server and trusts the test CA pool.
func tailnetHTTPSClient(srv *tsnet.Server) *http.Client {
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: testCAs},
DialContext: srv.Dial,
},
}
}
// nginxDeployment returns a simple nginx Deployment for use in tests.
func nginxDeployment(namespace, name string) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
@@ -428,7 +472,7 @@ func testIngressIsReachable(t *testing.T, httpClient *http.Client, url string) e
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
resp, err = httpClient.Do(req.WithContext(ctx))
if err != nil {
@@ -439,7 +483,6 @@ func testIngressIsReachable(t *testing.T, httpClient *http.Client, url string) e
}); err != nil {
return fmt.Errorf("error trying to reach %s: %w", url, err)
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status from %s: %d", url, resp.StatusCode)
}

138
cmd/k8s-operator/e2e/multitailnet_test.go
View File

@@ -1,138 +0,0 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package e2e
import (
"context"
"fmt"
"net/http"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tstest"
"tailscale.com/util/httpm"
)
// TestMultiTailnet verifies that ProxyGroup resources are created in the correct Tailnet,
// and that an Ingress resource has its Tailscale Service created in the correct Tailnet.
func TestMultiTailnet(t *testing.T) {
if tnClient == nil || secondTSClient == nil {
t.Skip("TestMultiTailnet requires a working tailnet client for a primary and second tailnet")
}
// Apply nginx Deployment and Service.
createAndCleanup(t, kubeClient, nginxDeployment("default", "nginx"))
createAndCleanup(t, kubeClient, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "nginx",
Namespace: "default",
},
Spec: corev1.ServiceSpec{
Selector: map[string]string{
"app.kubernetes.io/name": "nginx",
},
Ports: []corev1.ServicePort{
{
Name: "http",
Port: 80,
},
},
},
})
// Create Ingress ProxyGroup for each Tailnet.
firstTailnetPG := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "first-tailnet",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
},
}
createAndCleanup(t, kubeClient, firstTailnetPG)
secondTailnetPG := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "second-tailnet",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
Tailnet: "second-tailnet",
},
}
createAndCleanup(t, kubeClient, secondTailnetPG)
// TODO: Verify that devices have been created in the expected Tailnet.
// Apply Ingress to expose nginx.
ingress := &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "second-tailnet",
Namespace: "default",
Annotations: map[string]string{
"tailscale.com/proxy-group": "second-tailnet",
},
},
Spec: networkingv1.IngressSpec{
IngressClassName: new("tailscale"),
TLS: []networkingv1.IngressTLS{
networkingv1.IngressTLS{
Hosts: []string{"second-tailnet"},
},
},
Rules: []networkingv1.IngressRule{
{
IngressRuleValue: networkingv1.IngressRuleValue{
HTTP: &networkingv1.HTTPIngressRuleValue{
Paths: []networkingv1.HTTPIngressPath{
{
Path: "/",
PathType: new(networkingv1.PathTypePrefix),
Backend: networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: "nginx",
Port: networkingv1.ServiceBackendPort{
Number: 80,
},
},
},
},
},
},
},
},
},
},
}
createAndCleanup(t, kubeClient, ingress)
hostname, err := waitForIngressHostname(t, "default", "second-tailnet")
if err != nil {
t.Fatalf("error waiting for Ingress hostname: %v", err)
}
httpClient := tailnetHTTPSClient(secondTNClient)
var resp *http.Response
if err := tstest.WaitFor(time.Minute, func() error {
req, err := http.NewRequest(httpm.GET, fmt.Sprintf("https://%s:443", hostname), nil)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
resp, err = httpClient.Do(req.WithContext(ctx))
return err
}); err != nil {
t.Fatalf("error trying to reach Ingress: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %v", resp.StatusCode)
}
}

188
cmd/k8s-operator/e2e/setup.go
View File

@@ -263,116 +263,45 @@ func runTests(m *testing.M) (int, error) {
tsClient.BaseURL = "http://localhost:31544"
// Set ACLS and create Oauth client for primary tailnet
req, _ := http.NewRequestWithContext(ctx, "POST", tsClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs))
resp, err := tsClient.Do(req)
err = setACLs(ctx, tsClient)
if err != nil {
return 0, fmt.Errorf("failed to set ACLs: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return 0, fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b))
return 0, err
}
logger.Info("set ACLs for primary tailnet")
reqBody, err := json.Marshal(map[string]any{
"keyType": "client",
"scopes": []string{"auth_keys", "devices:core", "services"},
"tags": []string{"tag:k8s-operator"},
"description": "k8s-operator client for e2e tests",
})
if err != nil {
return 0, fmt.Errorf("failed to marshal OAuth client creation request: %w", err)
}
req, _ = http.NewRequestWithContext(ctx, "POST", tsClient.BuildTailnetURL("keys"), bytes.NewReader(reqBody))
resp, err = tsClient.Do(req)
if err != nil {
return 0, fmt.Errorf("failed to create OAuth client: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return 0, fmt.Errorf("HTTP %d creating OAuth client: %s", resp.StatusCode, string(b))
}
var key struct {
ID string `json:"id"`
Key string `json:"key"`
}
if err := json.NewDecoder(resp.Body).Decode(&key); err != nil {
return 0, fmt.Errorf("failed to decode OAuth client creation response: %w", err)
}
clientID = key.ID
clientSecret = key.Key
clientID, clientSecret, err = createOauthCreds(ctx, tsClient)
logger.Info("set Oauth credentials for primary tailnet")
// Create second tailnet. The bootstrap credentials returned have all permissions-
// they are used only to create an OAuth client for the k8s-operator
bootstrapID, bootstrapSecret, err := createTailnet(ctx, tsClient)
// they are used only to create an OAuth client for the k8s-operator.
bootstrapSecret, err := createTailnet(ctx, tsClient)
if err != nil {
return 0, fmt.Errorf("failed to create second tailnet: %w", err)
}
bootstrapClient, err := oauthTSClient(ctx, "http://localhost:31544", bootstrapID, bootstrapSecret)
bootstrapClient, err := oauthTSClient(ctx, "http://localhost:31544", bootstrapSecret)
if err != nil {
return 0, fmt.Errorf("failed to set up bootstrap client for second tailnet: %w", err)
}
// Set HTTPS on second tailnet.
req, _ = http.NewRequestWithContext(ctx, http.MethodPatch, bootstrapClient.BuildTailnetURL("settings"),
bytes.NewBufferString(`{"httpsEnabled": true}`))
resp, err = bootstrapClient.Do(req)
err = setHTTPSForTailnet(ctx, bootstrapClient)
if err != nil {
return 0, fmt.Errorf("failed to enable HTTPS: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return 0, fmt.Errorf("HTTP %d enabling HTTPS: %s", resp.StatusCode, string(b))
return 0, err
}
logger.Info("HTTPS settings configured for second tailnet")
// Set ACLs for second tailnet.
req, _ = http.NewRequestWithContext(ctx, http.MethodPost, bootstrapClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs))
resp, err = bootstrapClient.Do(req)
err = setACLs(ctx, bootstrapClient)
if err != nil {
return 0, fmt.Errorf("failed to set ACLs: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return 0, fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b))
return 0, err
}
logger.Info("set ACLs for second tailnet")
// Create an OAuth client for the second tailnet with the same
// scopes and tag as the primary tailnet's k8s-operator.
reqBody, err = json.Marshal(map[string]any{
"keyType": "client",
"scopes": []string{"auth_keys", "devices:core", "services"},
"tags": []string{"tag:k8s-operator"},
"description": "k8s-operator client for e2e tests",
})
if err != nil {
return 0, fmt.Errorf("failed to marshal OAuth client creation request: %w", err)
}
req, _ = http.NewRequestWithContext(ctx, http.MethodPost, bootstrapClient.BuildTailnetURL("keys"), bytes.NewReader(reqBody))
resp, err = bootstrapClient.Do(req)
if err != nil {
return 0, fmt.Errorf("failed to create OAuth client for second tailnet: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return 0, fmt.Errorf("HTTP %d creating OAuth client for second tailnet: %s", resp.StatusCode, string(b))
}
var secondKey struct {
ID string `json:"id"`
Key string `json:"key"`
}
if err := json.NewDecoder(resp.Body).Decode(&secondKey); err != nil {
return 0, fmt.Errorf("failed to decode OAuth client creation response: %w", err)
}
secondClientID = secondKey.ID
secondClientSecret = secondKey.Key
secondClientID, secondClientSecret, err = createOauthCreds(ctx, bootstrapClient)
logger.Info("set OAuth credentials for second tailnet")
secondTSClient, err = oauthTSClient(ctx, "http://localhost:31544", secondClientID, secondClientSecret)
secondTSClient, err = oauthTSClient(ctx, "http://localhost:31544", secondClientSecret)
if err != nil {
return 0, fmt.Errorf("failed to set up second tailnet client: %w", err)
}
@@ -382,31 +311,30 @@ func runTests(m *testing.M) (int, error) {
if clientSecret == "" {
return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator")
}
// Format is "tskey-client-<id>-<random>".
// TODO: remove duplication
parts := strings.Split(clientSecret, "-")
if len(parts) != 4 {
return 0, fmt.Errorf("TS_API_CLIENT_SECRET is not valid")
}
clientID = parts[2]
tsClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, clientID, clientSecret)
tsClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, clientSecret)
if err != nil {
return 0, fmt.Errorf("failed to set up primary tailnet client: %w", err)
}
secondClientSecret = os.Getenv("SECOND_TS_API_CLIENT_SECRET")
if secondClientSecret == "" {
return 0, fmt.Errorf("must use --devcontrol or set SECOND_TS_API_CLIENT_SECRET to an OAuth client suitable for the operator")
}
// Format is "tskey-client-<id>-<random>".
parts = strings.Split(secondClientSecret, "-")
if len(parts) != 4 {
return 0, fmt.Errorf("SECOND_TS_API_CLIENT_SECRET is not valid")
}
secondClientID = parts[2]
secondTSClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, secondClientID, secondClientSecret)
secondTSClient, err = oauthTSClient(ctx, ipn.DefaultControlURL, secondClientSecret)
if err != nil {
return 0, fmt.Errorf("failed to set up second tailnet client: %w", err)
}
}
var ossTag string
@@ -621,6 +549,65 @@ func runTests(m *testing.M) (int, error) {
return m.Run(), nil
}
func setHTTPSForTailnet(ctx context.Context, client *tailscale.Client) error {
req, _ := http.NewRequestWithContext(ctx, http.MethodPatch, client.BuildTailnetURL("settings"),
bytes.NewBufferString(`{"httpsEnabled": true}`))
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to enable HTTPS: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return fmt.Errorf("HTTP %d enabling HTTPS: %s", resp.StatusCode, string(b))
}
return nil
}
func setACLs(ctx context.Context, client *tailscale.Client) error {
req, _ := http.NewRequestWithContext(ctx, "POST", tsClient.BuildTailnetURL("acl"), bytes.NewReader(requiredACLs))
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to set ACLs: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return fmt.Errorf("HTTP %d setting ACLs: %s", resp.StatusCode, string(b))
}
return nil
}
func createOauthCreds(ctx context.Context, client *tailscale.Client) (string, string, error) {
reqBody, err := json.Marshal(map[string]any{
"keyType": "client",
"scopes": []string{"auth_keys", "devices:core", "services"},
"tags": []string{"tag:k8s-operator"},
"description": "k8s-operator client for e2e tests",
})
if err != nil {
return "", "", fmt.Errorf("failed to marshal OAuth client creation request: %w", err)
}
req, _ := http.NewRequestWithContext(ctx, "POST", client.BuildTailnetURL("keys"), bytes.NewReader(reqBody))
resp, err := tsClient.Do(req)
if err != nil {
return "", "", fmt.Errorf("failed to create OAuth client: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return "", "", fmt.Errorf("HTTP %d creating OAuth client: %s", resp.StatusCode, string(b))
}
var key struct {
ID string `json:"id"`
Key string `json:"key"`
}
if err := json.NewDecoder(resp.Body).Decode(&key); err != nil {
return "", "", fmt.Errorf("failed to decode OAuth client creation response: %w", err)
}
return key.ID, key.Key, nil
}
func upgraderOrInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc {
hist := action.NewHistory(cfg)
hist.Max = 1
@@ -878,25 +865,25 @@ func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts
return nil
}
func createTailnet(ctx context.Context, cl *tailscale.Client) (clientID, clientSecret string, err error) {
func createTailnet(ctx context.Context, cl *tailscale.Client) (clientSecret string, err error) {
tailnetName := fmt.Sprintf("second-tailnet-%d", time.Now().Unix())
body, err := json.Marshal(map[string]any{"displayName": tailnetName})
if err != nil {
return "", "", err
return "", err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
cl.BaseURL+"/api/v2/organizations/-/tailnets", bytes.NewBuffer(body))
if err != nil {
return "", "", err
return "", err
}
resp, err := cl.Do(req)
if err != nil {
return "", "", err
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return "", "", fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b))
return "", fmt.Errorf("HTTP %d creating tailnet: %s", resp.StatusCode, string(b))
}
var result struct {
OauthClient struct {
@@ -905,15 +892,20 @@ func createTailnet(ctx context.Context, cl *tailscale.Client) (clientID, clientS
} `json:"oauthClient"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", "", fmt.Errorf("failed to decode response: %w", err)
return "", fmt.Errorf("failed to decode response: %w", err)
}
return result.OauthClient.ID, result.OauthClient.Secret, nil
return result.OauthClient.Secret, nil
}
// oauthTSClient exchanges OAuth client credentials for an access token and
// returns a tailscale.Client configured to use it. The token is valid for
// one hour, which is sufficient for a single test run.
func oauthTSClient(ctx context.Context, baseURL, clientID, clientSecret string) (*tailscale.Client, error) {
func oauthTSClient(ctx context.Context, baseURL, clientSecret string) (*tailscale.Client, error) {
parts := strings.Split(clientSecret, "-")
if len(parts) != 4 {
return nil, fmt.Errorf("TS_API_CLIENT_SECRET is not valid")
}
clientID := parts[2]
cfg := clientcredentials.Config{
ClientID: clientID,
ClientSecret: clientSecret,

81
cmd/k8s-operator/ingress-for-pg.go
View File

@@ -138,19 +138,16 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque
// to the Tailscale Service in a multi-cluster Ingress setup have not
// resulted in another actor overwriting our Tailscale Service update.
needsRequeue := false
shortRequeue := false
if !ing.DeletionTimestamp.IsZero() || !r.shouldExpose(ing) {
needsRequeue, err = r.maybeCleanup(ctx, hostname, ing, logger, tailscaleClient, pg)
} else {
needsRequeue, shortRequeue, err = r.maybeProvision(ctx, hostname, ing, logger, tailscaleClient, pg)
needsRequeue, err = r.maybeProvision(ctx, hostname, ing, logger, tailscaleClient, pg)
}
if err != nil {
return res, err
}
if needsRequeue {
res = reconcile.Result{RequeueAfter: requeueInterval()}
} else if shortRequeue {
res = reconcile.Result{RequeueAfter: 30 * time.Second}
}
return res, nil
}
@@ -163,37 +160,37 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque
// If a Tailscale Service exists, but does not have an owner reference from any operator, we error
// out assuming that this is an owner reference created by an unknown actor.
// Returns true if the operation resulted in a Tailscale Service update.
func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsClient, pg *tsapi.ProxyGroup) (svcsChanged bool, shortRequeue bool, err error) {
func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger, tsClient tsClient, pg *tsapi.ProxyGroup) (svcsChanged bool, err error) {
// Currently (2025-05) Tailscale Services are behind an alpha feature flag that
// needs to be explicitly enabled for a tailnet to be able to use them.
serviceName := tailcfg.ServiceName("svc:" + hostname)
existingTSSvc, err := tsClient.GetVIPService(ctx, serviceName)
if err != nil && !isErrorTailscaleServiceNotFound(err) {
return false, false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err)
return false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err)
}
if err = validateIngressClass(ctx, r.Client, r.ingressClassName); err != nil {
logger.Infof("error validating tailscale IngressClass: %v.", err)
return false, false, nil
return false, nil
}
// Get and validate ProxyGroup readiness
pgName := ing.Annotations[AnnotationProxyGroup]
if pgName == "" {
logger.Infof("[unexpected] no ProxyGroup annotation, skipping Tailscale Service provisioning")
return false, false, nil
return false, nil
}
logger = logger.With("ProxyGroup", pgName)
if !tsoperator.ProxyGroupAvailable(pg) {
logger.Infof("ProxyGroup is not (yet) ready")
return false, false, nil
return false, nil
}
// Validate Ingress configuration
if err := r.validateIngress(ctx, ing, pg); err != nil {
logger.Infof("invalid Ingress configuration: %v", err)
r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidIngressConfiguration", err.Error())
return false, false, nil
return false, nil
}
if !IsHTTPSEnabledOnTailnet(r.tsnetServer) {
@@ -208,7 +205,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
logger.Infof("exposing Ingress over tailscale")
ing.Finalizers = append(ing.Finalizers, FinalizerNamePG)
if err := r.Update(ctx, ing); err != nil {
return false, false, fmt.Errorf("failed to add finalizer: %w", err)
return false, fmt.Errorf("failed to add finalizer: %w", err)
}
r.mu.Lock()
r.managedIngresses.Add(ing.UID)
@@ -226,7 +223,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
// (eventually) removed.
svcsChanged, err = r.maybeCleanupProxyGroup(ctx, logger, tsClient, pg)
if err != nil {
return false, false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err)
return false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err)
}
// 2. Ensure that there isn't a Tailscale Service with the same hostname
@@ -246,31 +243,31 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr)
logger.Warn(msg)
r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidTailscaleService", msg)
return false, false, nil
return false, nil
}
// 3. Ensure that TLS Secret and RBAC exists
dnsName, err := dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace)
if err != nil {
return false, false, fmt.Errorf("error determining DNS name for service: %w", err)
return false, fmt.Errorf("error determining DNS name for service: %w", err)
}
if err = r.ensureCertResources(ctx, pg, dnsName, ing); err != nil {
return false, false, fmt.Errorf("error ensuring cert resources: %w", err)
return false, fmt.Errorf("error ensuring cert resources: %w", err)
}
// 4. Ensure that the serve config for the ProxyGroup contains the Tailscale Service.
cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName)
if err != nil {
return false, false, fmt.Errorf("error getting Ingress serve config: %w", err)
return false, fmt.Errorf("error getting Ingress serve config: %w", err)
}
if cm == nil {
logger.Infof("no Ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.")
return svcsChanged, shortRequeue, nil
return svcsChanged, nil
}
ep := ipn.HostPort(fmt.Sprintf("%s:443", dnsName))
handlers, err := handlersForIngress(ctx, ing, r.Client, r.recorder, dnsName, logger)
if err != nil {
return false, false, fmt.Errorf("failed to get handlers for Ingress: %w", err)
return false, fmt.Errorf("failed to get handlers for Ingress: %w", err)
}
ingCfg := &ipn.ServiceConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
@@ -325,11 +322,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
mak.Set(&cfg.Services, serviceName, ingCfg)
cfgBytes, err := json.Marshal(cfg)
if err != nil {
return false, false, fmt.Errorf("error marshaling serve config: %w", err)
return false, fmt.Errorf("error marshaling serve config: %w", err)
}
mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
if err := r.Update(ctx, cm); err != nil {
return false, false, fmt.Errorf("error updating serve config: %w", err)
return false, fmt.Errorf("error updating serve config: %w", err)
}
}
@@ -363,7 +360,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
!ownersAreSetAndEqual(tsSvc, existingTSSvc) {
logger.Infof("Ensuring Tailscale Service exists and is up to date")
if err := tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil {
return false, false, fmt.Errorf("error creating Tailscale Service: %w", err)
return false, fmt.Errorf("error creating Tailscale Service: %w", err)
}
}
@@ -373,19 +370,14 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
if isHTTPEndpointEnabled(ing) || isHTTPRedirectEnabled(ing) {
mode = serviceAdvertisementHTTPAndHTTPS
}
shouldBeAdvertised, err := r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg)
if err != nil {
return false, false, fmt.Errorf("failed to update tailscaled config: %w", err)
}
// If certs are not yet ready, schedule a short requeue.
if !shouldBeAdvertised {
shortRequeue = true
if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, mode, pg); err != nil {
return false, fmt.Errorf("failed to update tailscaled config: %w", err)
}
// 6. Update Ingress status if ProxyGroup Pods are ready.
count, err := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName)
if err != nil {
return false, false, fmt.Errorf("failed to check if any Pods are configured: %w", err)
return false, fmt.Errorf("failed to check if any Pods are configured: %w", err)
}
oldStatus := ing.Status.DeepCopy()
@@ -393,14 +385,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
switch count {
case 0:
ing.Status.LoadBalancer.Ingress = nil
if shouldBeAdvertised {
shortRequeue = true
}
default:
var ports []networkingv1.IngressPortStatus
hasCerts, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg)
if err != nil {
return false, false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err)
return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err)
}
// If TLS certs have not been issued (yet), do not set port 443.
if hasCerts {
@@ -428,7 +417,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
}
}
if apiequality.Semantic.DeepEqual(oldStatus, &ing.Status) {
return svcsChanged, shortRequeue, nil
return svcsChanged, nil
}
const prefix = "Updating Ingress status"
@@ -439,10 +428,10 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
}
if err = r.Status().Update(ctx, ing); err != nil {
return false, false, fmt.Errorf("failed to update Ingress status: %w", err)
return false, fmt.Errorf("failed to update Ingress status: %w", err)
}
return svcsChanged, shortRequeue, nil
return svcsChanged, nil
}
// maybeCleanupProxyGroup ensures that any Tailscale Services that are
@@ -496,7 +485,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, logger
}
// Make sure the Tailscale Service is not advertised in tailscaled or serve config.
if _, err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil {
if err = r.maybeUpdateAdvertiseServicesConfig(ctx, tsSvcName, serviceAdvertisementOff, pg); err != nil {
return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
}
@@ -582,7 +571,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
}
// 4. Unadvertise the Tailscale Service in tailscaled config.
if _, err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil {
if err = r.maybeUpdateAdvertiseServicesConfig(ctx, serviceName, serviceAdvertisementOff, pg); err != nil {
return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
}
@@ -766,11 +755,11 @@ func isHTTPEndpointEnabled(ing *networkingv1.Ingress) bool {
serviceAdvertisementHTTPAndHTTPS // Both ports 80 and 443 should be advertised
)
func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup) (shouldBeAdvertised bool, err error) {
func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, pg *tsapi.ProxyGroup) (err error) {
// Get all config Secrets for this ProxyGroup.
secrets := &corev1.SecretList{}
if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil {
return false, fmt.Errorf("failed to list config Secrets: %w", err)
return fmt.Errorf("failed to list config Secrets: %w", err)
}
// Verify that TLS cert for the Tailscale Service has been successfully issued
@@ -783,9 +772,9 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
// TLS cert is not yet provisioned.
hasCert, err := hasCerts(ctx, r.Client, r.tsNamespace, serviceName, pg)
if err != nil {
return false, fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err)
return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err)
}
shouldBeAdvertised = (mode == serviceAdvertisementHTTPAndHTTPS) ||
shouldBeAdvertised := (mode == serviceAdvertisementHTTPAndHTTPS) ||
(mode == serviceAdvertisementHTTPS && hasCert) // if we only expose port 443 and don't have certs (yet), do not advertise
for _, secret := range secrets.Items {
@@ -793,7 +782,7 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
for fileName, confB := range secret.Data {
var conf ipn.ConfigVAlpha
if err := json.Unmarshal(confB, &conf); err != nil {
return false, fmt.Errorf("error unmarshalling ProxyGroup config: %w", err)
return fmt.Errorf("error unmarshalling ProxyGroup config: %w", err)
}
// Update the services to advertise if required.
@@ -814,7 +803,7 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
// Update the Secret.
confB, err := json.Marshal(conf)
if err != nil {
return false, fmt.Errorf("error marshalling ProxyGroup config: %w", err)
return fmt.Errorf("error marshalling ProxyGroup config: %w", err)
}
mak.Set(&secret.Data, fileName, confB)
updated = true
@@ -822,12 +811,12 @@ func (r *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
if updated {
if err := r.Update(ctx, &secret); err != nil {
return false, fmt.Errorf("error updating ProxyGroup config Secret: %w", err)
return fmt.Errorf("error updating ProxyGroup config Secret: %w", err)
}
}
}
return shouldBeAdvertised, nil
return nil
}
func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, pgName string, serviceName tailcfg.ServiceName) (int, error) {