mirror/tailscale
1
0 Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2026-09-14 23:11:39 -04:00
Code Issues Packages Projects Releases Wiki Activity

cmd/k8s-operator: move egress reconcilers into their own package

Browse Source 
Move the four egress reconcilers into k8s-operator/reconciler/egress,
following the earlier ProxyClass, DNSConfig, nameserver and Recorder moves.
They register themselves, so operator.go now builds every packaged
reconciler into a reconciler.Reconciler slice and registers them in a loop.

Hoist the helpers they shared with the rest of the operator: the Service
annotations, validation and cluster domain lookup into reconciler, and
pgReplicas into ProxyGroupReplicas. The per-CRD condition helpers move to
reconciler too, on a shared SetCondition, and the ones only used by their
own reconciler become unexported there. Unused condition helpers are
deleted.

This takes package kube down to the contract it shares with containerboot
and k8s-nameserver, which no longer link controller-runtime; containerboot
drops from 25MB to 15MB and k8s-nameserver from 20MB to 9MB.

Updates tailscale/corp#46764

Signed-off-by: David Bond <davidsbond93@gmail.com>
This commit is contained in:
David Bond committed 2026-08-18 17:46:25 +01:00
1 parent 7ee121a0fb
commit 950eb069b9
44 files changed
+2039 -1687
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmd/k8s-operator/api-server-proxy-pg.go
+6 -6
View File
@@ -25,8 +25,8 @@
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/client/tailscale/v2"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
@@ -123,7 +123,7 @@ func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, s
// Update the condition based on how many pods are advertising the service
conditionStatus := metav1.ConditionFalse
conditionReason := reasonKubeAPIServerProxyNoBackends
conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, pgReplicas(pg))
conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, reconciler.ProxyGroupReplicas(pg))
pg.Status.URL = ""
if podsAdvertising > 0 {
@@ -135,7 +135,7 @@ func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, s
}
}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger)
if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) {
// An error encountered here should get returned by the Reconcile function.
@@ -143,7 +143,7 @@ func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, s
}
}()
if !tsoperator.ProxyGroupAvailable(pg) {
if !reconciler.ProxyGroupAvailable(pg) {
return nil
}
@@ -172,12 +172,12 @@ func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, s
msg := fmt.Sprintf("error ensuring exclusive ownership of Tailscale Service %s: %v. %s", serviceName, err, instr)
logger.Warn(msg)
r.recorder.Event(pg, corev1.EventTypeWarning, "InvalidTailscaleService", msg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger)
return nil
}
// After getting this far, we know the Tailscale Service is valid.
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger)
// Service tags are limited to matching the ProxyGroup's tags until we have
// support for querying peer caps for a Service-bound request.
cmd/k8s-operator/api-server-proxy-pg_test.go
+8 -8
View File
@@ -18,8 +18,8 @@
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"tailscale.com/client/tailscale/v2"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/k8s-proxy/conf"
@@ -130,8 +130,8 @@ func TestAPIServerProxyReconciler(t *testing.T) {
}
expectReconciled(t, r, "", pgName)
pg.ObjectMeta.Finalizers = []string{proxyPGFinalizerName}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
expectMissing[corev1.Secret](t, fc, ns, defaultDomain)
expectMissing[rbacv1.Role](t, fc, ns, defaultDomain)
@@ -178,8 +178,8 @@ func TestAPIServerProxyReconciler(t *testing.T) {
if !reflect.DeepEqual(tsSvc, expectedTSSvc) {
t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc)
}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
expectedCfg.APIServerProxy.ServiceName = new(tailcfg.ServiceName("svc:" + pgName))
@@ -213,7 +213,7 @@ func TestAPIServerProxyReconciler(t *testing.T) {
})
expectReconciled(t, r, "", pgName)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
pg.Status.URL = "https://" + defaultDomain
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
@@ -243,7 +243,7 @@ func TestAPIServerProxyReconciler(t *testing.T) {
expectedCfg.APIServerProxy.ServiceName = new(updatedServiceName)
expectedCfg.AdvertiseServices = nil
expectCfg(&expectedCfg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
pg.Status.URL = ""
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
@@ -262,7 +262,7 @@ func TestAPIServerProxyReconciler(t *testing.T) {
expectReconciled(t, r, "", pgName)
expectedCfg.AdvertiseServices = []string{updatedServiceName.String()}
expectCfg(&expectedCfg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
pg.Status.URL = "https://" + updatedDomain
// Delete the ProxyGroup and verify Tailscale Service and cert resources are cleaned up.
cmd/k8s-operator/connector.go
+2 -2
View File
@@ -26,8 +26,8 @@
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
"tailscale.com/net/netutil"
"tailscale.com/net/tsaddr"
@@ -115,7 +115,7 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
oldCnStatus := cn.Status.DeepCopy()
setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
reconciler.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
var updateErr error
if !apiequality.Semantic.DeepEqual(oldCnStatus, &cn.Status) {
// An error encountered here should get returned by the Reconcile function.
cmd/k8s-operator/depaware.txt
+6 -5
View File
@@ -353,7 +353,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/apiserver/pkg/authentication/user from k8s.io/apiserver/pkg/endpoints/request
k8s.io/apiserver/pkg/endpoints/request from tailscale.com/k8s-operator/api-proxy
k8s.io/apiserver/pkg/features from k8s.io/apiserver/pkg/endpoints/request
k8s.io/apiserver/pkg/storage/names from tailscale.com/cmd/k8s-operator
k8s.io/apiserver/pkg/storage/names from tailscale.com/cmd/k8s-operator+
k8s.io/apiserver/pkg/util/feature from k8s.io/apiserver/pkg/endpoints/request+
k8s.io/client-go/applyconfigurations/admissionregistration/v1 from k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1+
k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1
@@ -769,6 +769,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+
tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet+
tailscale.com/k8s-operator/reconciler/dnsrecords from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/egress from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/nameserver from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/peerrelay from tailscale.com/cmd/k8s-operator
tailscale.com/k8s-operator/reconciler/proxyclass from tailscale.com/cmd/k8s-operator
@@ -781,7 +782,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/k8s-operator/sessionrecording/tsrecorder from tailscale.com/k8s-operator/sessionrecording+
tailscale.com/k8s-operator/sessionrecording/ws from tailscale.com/k8s-operator/sessionrecording
tailscale.com/k8s-operator/tsclient from tailscale.com/cmd/k8s-operator+
tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator
tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator+
tailscale.com/kube/ingressservices from tailscale.com/cmd/k8s-operator
tailscale.com/kube/k8s-proxy/conf from tailscale.com/cmd/k8s-operator
tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+
@@ -798,7 +799,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
💣 tailscale.com/net/batching from tailscale.com/wgengine/magicsock
tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+
tailscale.com/net/dns/publicdns from tailscale.com/net/dns+
tailscale.com/net/dns/resolvconffile from tailscale.com/cmd/k8s-operator+
tailscale.com/net/dns/resolvconffile from tailscale.com/k8s-operator/reconciler+
tailscale.com/net/dns/resolver from tailscale.com/net/dns+
tailscale.com/net/dnscache from tailscale.com/control/controlclient+
tailscale.com/net/dnsfallback from tailscale.com/control/controlclient+
@@ -879,7 +880,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/types/structs from tailscale.com/control/controlclient+
tailscale.com/types/tkatype from tailscale.com/client/local+
tailscale.com/types/views from tailscale.com/appc+
tailscale.com/util/backoff from tailscale.com/cmd/k8s-operator+
tailscale.com/util/backoff from tailscale.com/control/controlclient+
tailscale.com/util/bufiox from tailscale.com/types/key
tailscale.com/util/checkchange from tailscale.com/ipn/ipnlocal+
tailscale.com/util/cibuild from tailscale.com/health+
@@ -959,7 +960,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+
golang.org/x/exp/constraints from tailscale.com/tsweb/varz+
golang.org/x/exp/maps from tailscale.com/ipn/store/mem+
golang.org/x/exp/slices from tailscale.com/cmd/k8s-operator+
golang.org/x/exp/slices from tailscale.com/cmd/k8s-operator
golang.org/x/net/bpf from github.com/mdlayher/netlink+
golang.org/x/net/dns/dnsmessage from tailscale.com/appc+
golang.org/x/net/http/httpguts from golang.org/x/net/http2+
cmd/k8s-operator/e2e/egress_test.go
+5 -5
View File
@@ -11,8 +11,8 @@
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kube "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/tstest"
)
@@ -33,7 +33,7 @@ func TestEgress(t *testing.T) {
"tailscale.com/tailnet-ip": tnTarget.ipv4,
})
createAndCleanup(t, kubeClient, svc)
waitForEgress(t, svc.Name, kube.SvcIsReady)
waitForEgress(t, svc.Name, reconciler.SvcIsReady)
testEgressIsReachable(t, ns, svc.Name)
})
@@ -45,7 +45,7 @@ func TestEgress(t *testing.T) {
"tailscale.com/tailnet-ip": tnTarget.ipv6,
})
createAndCleanup(t, kubeClient, svc)
waitForEgress(t, svc.Name, kube.SvcIsReady)
waitForEgress(t, svc.Name, reconciler.SvcIsReady)
testEgressIsReachable(t, ns, svc.Name)
})
@@ -54,7 +54,7 @@ func TestEgress(t *testing.T) {
"tailscale.com/tailnet-fqdn": tnTarget.fqdn,
})
createAndCleanup(t, kubeClient, svc)
waitForEgress(t, svc.Name, kube.SvcIsReady)
waitForEgress(t, svc.Name, reconciler.SvcIsReady)
testEgressIsReachable(t, ns, svc.Name)
})
}
@@ -165,7 +165,7 @@ func egressService(name string, annotations map[string]string) *corev1.Service {
}
func pgEgressReady(svc *corev1.Service) bool {
cond := kube.GetServiceCondition(svc, tsapi.EgressSvcReady)
cond := reconciler.GetServiceCondition(svc, tsapi.EgressSvcReady)
return cond != nil && cond.Status == metav1.ConditionTrue
}
cmd/k8s-operator/e2e/ingress_test.go
+2 -2
View File
@@ -18,8 +18,8 @@
"sigs.k8s.io/controller-runtime/pkg/client"
"tailscale.com/client/tailscale/v2"
kube "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/util/httpm"
@@ -64,7 +64,7 @@ func TestL3Ingress(t *testing.T) {
if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil {
return err
}
isReady := kube.SvcIsReady(maybeReadySvc)
isReady := reconciler.SvcIsReady(maybeReadySvc)
if isReady {
t.Log("Service is ready")
return nil
cmd/k8s-operator/e2e/setup.go
+2 -2
View File
@@ -61,8 +61,8 @@
"tailscale.com/client/tailscale/v2"
"tailscale.com/ipn"
"tailscale.com/ipn/store/mem"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/tsnet"
"tailscale.com/util/must"
)
@@ -729,7 +729,7 @@ func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl c
if err := cl.Get(ctx, client.ObjectKeyFromObject(pc), pc); err != nil {
return fmt.Errorf("failed to get default ProxyClass: %w", err)
}
if tsoperator.ProxyClassIsReady(pc) {
if reconciler.ProxyClassIsReady(pc) {
break
}
logger.Info("waiting for default ProxyClass to be ready...")
cmd/k8s-operator/ingress-for-pg.go
+5 -24
View File
@@ -32,8 +32,8 @@
"tailscale.com/client/tailscale/v2"
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/kubetypes"
@@ -182,7 +182,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
}
logger = logger.With("ProxyGroup", pgName)
if !tsoperator.ProxyGroupAvailable(pg) {
if !reconciler.ProxyGroupAvailable(pg) {
logger.Infof("ProxyGroup is not (yet) ready")
return false, nil
}
@@ -679,7 +679,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki
var errs []error
// Validate tags if present
violations := tagViolations(ing)
violations := reconciler.TagViolations(ing)
if len(violations) > 0 {
errs = append(errs, fmt.Errorf("Ingress contains invalid tags: %v", strings.Join(violations, ",")))
}
@@ -702,7 +702,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki
}
// Validate ProxyGroup readiness
if !tsoperator.ProxyGroupAvailable(pg) {
if !reconciler.ProxyGroupAvailable(pg) {
errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name))
}
@@ -1108,7 +1108,7 @@ func certResourceLabels(pgName, domain string) map[string]string {
return map[string]string{
kubetypes.LabelManaged: "true",
labelProxyGroup: pgName,
labelDomain: tsoperator.TruncateLabelValue(domain),
labelDomain: reconciler.TruncateLabelValue(domain),
}
}
@@ -1135,22 +1135,3 @@ func hasCerts(ctx context.Context, cl client.Client, ns string, svc tailcfg.Serv
return len(cert) > 0 && len(key) > 0, nil
}
func tagViolations(obj client.Object) []string {
var violations []string
if obj == nil {
return nil
}
tags, ok := obj.GetAnnotations()[AnnotationTags]
if !ok {
return nil
}
for tag := range strings.SplitSeq(tags, ",") {
tag = strings.TrimSpace(tag)
if err := tailcfg.CheckTag(tag); err != nil {
violations = append(violations, fmt.Sprintf("invalid tag %q: %v", tag, err))
}
}
return violations
}
cmd/k8s-operator/metrics_resources.go
+6 -6
View File
@@ -20,8 +20,8 @@
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
kube "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
)
@@ -239,13 +239,13 @@ func metricsResourceLabels(opts *metricsOpts) map[string]string {
kubetypes.LabelManaged: "true",
labelMetricsTarget: opts.proxyStsName,
labelPromProxyType: opts.proxyType,
labelPromProxyParentName: kube.TruncateLabelValue(opts.proxyLabels[LabelParentName]),
labelPromProxyParentName: reconciler.TruncateLabelValue(opts.proxyLabels[LabelParentName]),
}
// Include namespace label for proxies created for a namespaced type.
if isNamespacedProxyType(opts.proxyType) {
lbls[labelPromProxyParentNamespace] = kube.TruncateLabelValue(opts.proxyLabels[LabelParentNamespace])
lbls[labelPromProxyParentNamespace] = reconciler.TruncateLabelValue(opts.proxyLabels[LabelParentNamespace])
}
lbls[labelPromJob] = kube.TruncateLabelValue(promJobName(opts))
lbls[labelPromJob] = reconciler.TruncateLabelValue(promJobName(opts))
return lbls
}
@@ -262,11 +262,11 @@ func promJobName(opts *metricsOpts) string {
func metricsSvcSelector(proxyLabels map[string]string, proxyType string) map[string]string {
sel := map[string]string{
labelPromProxyType: proxyType,
labelPromProxyParentName: kube.TruncateLabelValue(proxyLabels[LabelParentName]),
labelPromProxyParentName: reconciler.TruncateLabelValue(proxyLabels[LabelParentName]),
}
// Include namespace label for proxies created for a namespaced type.
if isNamespacedProxyType(proxyType) {
sel[labelPromProxyParentNamespace] = kube.TruncateLabelValue(proxyLabels[LabelParentNamespace])
sel[labelPromProxyParentNamespace] = reconciler.TruncateLabelValue(proxyLabels[LabelParentNamespace])
}
return sel
}
cmd/k8s-operator/operator.go
+70 -404
View File
@@ -10,9 +10,7 @@
import (
"context"
"fmt"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"time"
@@ -54,7 +52,9 @@
"tailscale.com/ipn/store/kubestore"
apiproxy "tailscale.com/k8s-operator/api-proxy"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/dnsrecords"
"tailscale.com/k8s-operator/reconciler/egress"
"tailscale.com/k8s-operator/reconciler/nameserver"
"tailscale.com/k8s-operator/reconciler/peerrelay"
"tailscale.com/k8s-operator/reconciler/proxyclass"
@@ -354,39 +354,82 @@ func runReconcilers(opts reconcilerOpts) {
}
clients := tsclient.NewProvider(tsclient.Wrap(opts.tsClient))
eventRecorder := mgr.GetEventRecorderFor("tailscale-operator")
tailnetOptions := tailnet.ReconcilerOptions{
// The four egress reconcilers cooperate on the same resources and so share one set of options.
egressOptions := egress.Options{
Client: mgr.GetClient(),
Recorder: eventRecorder,
TailscaleNamespace: opts.tailscaleNamespace,
OperatorSAName: opts.operatorSAName,
Logger: opts.log,
Clock: tstime.DefaultClock{},
Logger: opts.log,
Registry: clients,
}
if err = tailnet.NewReconciler(tailnetOptions).Register(mgr); err != nil {
startlog.Fatalf("could not register tailnet reconciler: %v", err)
// Reconcilers that live in their own packages under k8s-operator/reconciler register themselves, including the
// watches and field indexes they depend on, so they can all be set up in one loop. Registration order carries no
// meaning to controller-runtime. The reconcilers still defined in this package are wired up by hand below.
//
// TODO (irbekrm): switch to metadata-only watches for resources whose
// spec we don't need to inspect to reduce memory consumption.
// https://github.com/kubernetes-sigs/controller-runtime/issues/1159
reconcilers := []reconciler.Reconciler{
tailnet.NewReconciler(tailnet.ReconcilerOptions{
Client: mgr.GetClient(),
TailscaleNamespace: opts.tailscaleNamespace,
OperatorSAName: opts.operatorSAName,
Clock: tstime.DefaultClock{},
Logger: opts.log,
Registry: clients,
}),
proxygrouppolicy.NewReconciler(proxygrouppolicy.ReconcilerOptions{
Client: mgr.GetClient(),
}),
peerrelay.NewReconciler(peerrelay.ReconcilerOptions{
Client: mgr.GetClient(),
TailscaleNamespace: opts.tailscaleNamespace,
ProxyImage: opts.proxyImage,
DefaultTags: strings.Split(opts.proxyTags, ","),
Clients: clients,
Logger: opts.log,
}),
nameserver.NewReconciler(nameserver.ReconcilerOptions{
Client: mgr.GetClient(),
Recorder: eventRecorder,
TailscaleNamespace: opts.tailscaleNamespace,
Logger: opts.log,
Clock: tstime.DefaultClock{},
}),
proxyclass.NewReconciler(proxyclass.ReconcilerOptions{
Client: mgr.GetClient(),
Recorder: eventRecorder,
TsNamespace: opts.tailscaleNamespace,
Logger: opts.log,
Clock: tstime.DefaultClock{},
}),
dnsrecords.NewReconciler(dnsrecords.ReconcilerOptions{
Client: mgr.GetClient(),
TailscaleNamespace: opts.tailscaleNamespace,
Logger: opts.log,
IsDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
}),
recorder.NewReconciler(recorder.ReconcilerOptions{
Client: mgr.GetClient(),
Recorder: eventRecorder,
TailscaleNamespace: opts.tailscaleNamespace,
Clients: clients,
Logger: opts.log,
Clock: tstime.DefaultClock{},
}),
egress.NewReconciler(egressOptions),
egress.NewReadinessReconciler(egressOptions),
egress.NewEndpointSliceReconciler(egressOptions),
egress.NewPodReconciler(egressOptions),
}
proxyGroupPolicyOptions := proxygrouppolicy.ReconcilerOptions{
Client: mgr.GetClient(),
}
if err = proxygrouppolicy.NewReconciler(proxyGroupPolicyOptions).Register(mgr); err != nil {
startlog.Fatalf("could not register proxygrouppolicy reconciler: %v", err)
}
peerRelayOptions := peerrelay.ReconcilerOptions{
Client: mgr.GetClient(),
TailscaleNamespace: opts.tailscaleNamespace,
ProxyImage: opts.proxyImage,
DefaultTags: strings.Split(opts.proxyTags, ","),
Clients: clients,
Logger: opts.log,
}
if err = peerrelay.NewReconciler(peerRelayOptions).Register(mgr); err != nil {
startlog.Fatalf("could not register peerrelay reconciler: %v", err)
for _, r := range reconcilers {
if err = r.Register(mgr); err != nil {
startlog.Fatalf("could not register %T: %v", r, err)
}
}
svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler)
@@ -400,7 +443,6 @@ func runReconcilers(opts reconcilerOpts) {
opts.proxyActAsDefaultLoadBalancer,
))
eventRecorder := mgr.GetEventRecorderFor("tailscale-operator")
ssr := &tailscaleSTSReconciler{
Client: mgr.GetClient(),
tsnetServer: opts.tsServer,
@@ -557,131 +599,6 @@ func runReconcilers(opts reconcilerOpts) {
if err != nil {
startlog.Fatalf("could not create connector reconciler: %v", err)
}
// TODO (irbekrm): switch to metadata-only watches for resources whose
// spec we don't need to inspect to reduce memory consumption.
// https://github.com/kubernetes-sigs/controller-runtime/issues/1159
nameserverOptions := nameserver.ReconcilerOptions{
Client: mgr.GetClient(),
Recorder: eventRecorder,
TailscaleNamespace: opts.tailscaleNamespace,
Logger: opts.log,
Clock: tstime.DefaultClock{},
}
if err = nameserver.NewReconciler(nameserverOptions).Register(mgr); err != nil {
startlog.Fatalf("could not create nameserver reconciler: %v", err)
}
egressSvcFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsHandler)
egressProxyGroupFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsFromEgressProxyGroup(mgr.GetClient(), opts.log))
err = builder.
ControllerManagedBy(mgr).
Named("egress-svcs-reconciler").
Watches(&corev1.Service{}, egressSvcFilter).
Watches(&tsapi.ProxyGroup{}, egressProxyGroupFilter).
Complete(&egressSvcsReconciler{
Client: mgr.GetClient(),
tsNamespace: opts.tailscaleNamespace,
recorder: eventRecorder,
clock: tstime.DefaultClock{},
logger: opts.log.Named("egress-svcs-reconciler"),
})
if err != nil {
startlog.Fatalf("could not create egress Services reconciler: %v", err)
}
if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexEgressProxyGroup, indexEgressServices); err != nil {
startlog.Fatalf("failed setting up indexer for egress Services: %v", err)
}
egressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(egressSvcFromEps)
err = builder.
ControllerManagedBy(mgr).
Named("egress-svcs-readiness-reconciler").
Watches(&corev1.Service{}, egressSvcFilter).
Watches(&discoveryv1.EndpointSlice{}, egressSvcFromEpsFilter).
Complete(&egressSvcsReadinessReconciler{
Client: mgr.GetClient(),
tsNamespace: opts.tailscaleNamespace,
clock: tstime.DefaultClock{},
logger: opts.log.Named("egress-svcs-readiness-reconciler"),
})
if err != nil {
startlog.Fatalf("could not create egress Services readiness reconciler: %v", err)
}
epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler)
podsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGPods(mgr.GetClient(), opts.tailscaleNamespace))
secretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGStateSecrets(mgr.GetClient(), opts.tailscaleNamespace))
epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
err = builder.
ControllerManagedBy(mgr).
Named("egress-eps-reconciler").
Watches(&discoveryv1.EndpointSlice{}, epsFilter).
Watches(&corev1.Pod{}, podsFilter).
Watches(&corev1.Secret{}, secretsFilter).
Watches(&corev1.Service{}, epsFromExtNSvcFilter).
Complete(&egressEpsReconciler{
Client: mgr.GetClient(),
tsNamespace: opts.tailscaleNamespace,
logger: opts.log.Named("egress-eps-reconciler"),
})
if err != nil {
startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err)
}
podsForEps := handler.EnqueueRequestsFromMapFunc(podsFromEgressEps(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
podsER := handler.EnqueueRequestsFromMapFunc(egressPodsHandler)
err = builder.
ControllerManagedBy(mgr).
Named("egress-pods-readiness-reconciler").
Watches(&discoveryv1.EndpointSlice{}, podsForEps).
Watches(&corev1.Pod{}, podsER).
Complete(&egressPodsReconciler{
Client: mgr.GetClient(),
tsNamespace: opts.tailscaleNamespace,
clock: tstime.DefaultClock{},
logger: opts.log.Named("egress-pods-readiness-reconciler"),
httpClient: http.DefaultClient,
})
if err != nil {
startlog.Fatalf("could not create egress Pods readiness reconciler: %v", err)
}
proxyClassOptions := proxyclass.ReconcilerOptions{
Client: mgr.GetClient(),
Recorder: eventRecorder,
TsNamespace: opts.tailscaleNamespace,
Logger: opts.log,
Clock: tstime.DefaultClock{},
}
if err = proxyclass.NewReconciler(proxyClassOptions).Register(mgr); err != nil {
startlog.Fatalf("could not create proxyclass reconciler: %v", err)
}
dnsRecordsOptions := dnsrecords.ReconcilerOptions{
Client: mgr.GetClient(),
TailscaleNamespace: opts.tailscaleNamespace,
Logger: opts.log,
IsDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
}
if err = dnsrecords.NewReconciler(dnsRecordsOptions).Register(mgr); err != nil {
startlog.Fatalf("could not create DNS records reconciler: %v", err)
}
recorderOptions := recorder.ReconcilerOptions{
Client: mgr.GetClient(),
Recorder: eventRecorder,
TailscaleNamespace: opts.tailscaleNamespace,
Clients: clients,
Logger: opts.log,
Clock: tstime.DefaultClock{},
}
if err = recorder.NewReconciler(recorderOptions).Register(mgr); err != nil {
startlog.Fatalf("could not create Recorder reconciler: %v", err)
}
// kube-apiserver's Tailscale Service reconciler.
err = builder.
ControllerManagedBy(mgr).
@@ -1206,101 +1123,6 @@ func serviceHandler(_ context.Context, o client.Object) []reconcile.Request {
}
}
// isMagicDNSName reports whether name is a full tailnet node FQDN (with or
// without final dot).
func isMagicDNSName(name string) bool {
validMagicDNSName := regexp.MustCompile(`^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+\.ts\.net\.?$`)
return validMagicDNSName.MatchString(name)
}
// egressSvcsHandler returns accepts a Kubernetes object and returns a reconcile
// request for it , if the object is a Tailscale egress Service meant to be
// exposed on a ProxyGroup.
func egressSvcsHandler(_ context.Context, o client.Object) []reconcile.Request {
if !isEgressSvcForProxyGroup(o) {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: o.GetNamespace(),
Name: o.GetName(),
},
},
}
}
// egressEpsHandler returns accepts an EndpointSlice and, if the EndpointSlice
// is for an egress service, returns a reconcile request for it.
func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: o.GetNamespace(),
Name: o.GetName(),
},
},
}
}
func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request {
if typ := o.GetLabels()[LabelParentType]; typ != proxyTypeProxyGroup {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: o.GetNamespace(),
Name: o.GetName(),
},
},
}
}
// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is,
// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
return func(_ context.Context, o client.Object) []reconcile.Request {
if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
// have ingress ProxyGroups.
if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" {
return nil
}
pg, ok := o.GetLabels()[LabelParentName]
if !ok {
return nil
}
return reconcileRequestsForPG(pg, cl, ns)
}
}
// egressEpsFromPGStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is,
// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
return func(_ context.Context, o client.Object) []reconcile.Request {
if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
return nil
}
if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState {
return nil
}
pg, ok := o.GetLabels()[LabelParentName]
if !ok {
return nil
}
return reconcileRequestsForPG(pg, cl, ns)
}
}
func ingressSvcFromEps(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
svcName := o.GetLabels()[discoveryv1.LabelServiceName]
@@ -1333,52 +1155,6 @@ func ingressSvcFromEps(cl client.Client, logger *zap.SugaredLogger) handler.MapF
}
}
// egressSvcFromEps is an event handler for EndpointSlices. If an EndpointSlice is for an egress ExternalName Service
// meant to be exposed on a ProxyGroup, returns a reconcile request for the Service.
func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
return nil
}
if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
svcName, ok := o.GetLabels()[LabelParentName]
if !ok {
return nil
}
svcNs, ok := o.GetLabels()[LabelParentNamespace]
if !ok {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: svcNs,
Name: svcName,
},
},
}
}
func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.Request {
epsList := discoveryv1.EndpointSliceList{}
if err := cl.List(context.Background(), &epsList,
client.InNamespace(ns),
client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
return nil
}
reqs := make([]reconcile.Request, 0)
for _, ep := range epsList.Items {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: ep.Namespace,
Name: ep.Name,
},
})
}
return reqs
}
func isTLSSecret(secret *corev1.Secret) bool {
return secret.Type == corev1.SecretTypeTLS &&
secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" &&
@@ -1513,37 +1289,6 @@ func kubeAPIServerPGsFromSecret(cl client.Client, logger *zap.SugaredLogger) han
}
}
// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
// user-created ExternalName Services that should be exposed on this ProxyGroup.
func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
pg, ok := o.(*tsapi.ProxyGroup)
if !ok {
logger.Warn("ProxyGroup handler triggered for an object that is not a ProxyGroup")
return nil
}
if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
return nil
}
svcList := &corev1.ServiceList{}
if err := cl.List(ctx, svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil {
logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, svc := range svcList.Items {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: svc.Namespace,
Name: svc.Name,
},
})
}
return reqs
}
}
// ingressesFromIngressProxyGroup is an event handler for ingress ProxyGroups. It returns reconcile requests for all
// user-created Ingresses that should be exposed on this ProxyGroup.
func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -1575,76 +1320,6 @@ func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
}
}
// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
svc, ok := o.(*corev1.Service)
if !ok {
logger.Warn("Service handler triggered for an object that is not a Service")
return nil
}
if !isEgressSvcForProxyGroup(svc) {
return nil
}
epsList := &discoveryv1.EndpointSliceList{}
if err := cl.List(ctx, epsList, client.InNamespace(ns),
client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil {
logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, eps := range epsList.Items {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: eps.Namespace,
Name: eps.Name,
},
})
}
return reqs
}
}
func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
eps, ok := o.(*discoveryv1.EndpointSlice)
if !ok {
logger.Warn("EndpointSlice handler triggered for an object that is not a EndpointSlice")
return nil
}
if eps.Labels[labelProxyGroup] == "" {
return nil
}
if eps.Labels[labelSvcType] != "egress" {
return nil
}
podLabels := map[string]string{
kubetypes.LabelManaged: "true",
LabelParentType: "proxygroup",
LabelParentName: eps.Labels[labelProxyGroup],
}
podList := &corev1.PodList{}
if err := cl.List(ctx, podList, client.InNamespace(ns),
client.MatchingLabels(podLabels)); err != nil {
logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on EndpointSlice %s", err, eps.Name)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, pod := range podList.Items {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: pod.Namespace,
Name: pod.Name,
},
})
}
return reqs
}
}
// crdTransformer gets called before a CRD is stored to c/r cache, it removes the CRD spec to reduce memory consumption.
func crdTransformer(log *zap.SugaredLogger) toolscache.TransformFunc {
return func(o any) (any, error) {
@@ -1659,15 +1334,6 @@ func crdTransformer(log *zap.SugaredLogger) toolscache.TransformFunc {
}
}
// indexEgressServices adds a local index to cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
// index is used a list filter.
func indexEgressServices(o client.Object) []string {
if !isEgressSvcForProxyGroup(o) {
return nil
}
return []string{o.GetAnnotations()[AnnotationProxyGroup]}
}
// indexPGIngresses is used to select ProxyGroup-backed Services which are
// locally indexed in the cache for efficient listing without requiring labels.
func indexPGIngresses(o client.Object) []string {
cmd/k8s-operator/operator_test.go
+3 -79
View File
@@ -27,12 +27,11 @@
"tailscale.com/k8s-operator/apis/v1alpha1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/net/dns/resolvconffile"
"tailscale.com/tstest"
"tailscale.com/tstime"
"tailscale.com/util/dnsname"
"tailscale.com/util/mak"
)
@@ -1522,8 +1521,8 @@ func Test_isMagicDNSName(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isMagicDNSName(tt.in); got != tt.want {
t.Errorf("isMagicDNSName(%q) = %v, want %v", tt.in, got, tt.want)
if got := reconciler.IsMagicDNSName(tt.in); got != tt.want {
t.Errorf("reconciler.IsMagicDNSName(%q) = %v, want %v", tt.in, got, tt.want)
}
})
}
@@ -1753,72 +1752,6 @@ func Test_serviceHandlerForIngress_multipleIngressClasses(t *testing.T) {
}
}
func Test_clusterDomainFromResolverConf(t *testing.T) {
zl := zap.Must(zap.NewDevelopment())
tests := []struct {
name string
conf *resolvconffile.Config
namespace string
want string
}{
{
name: "success-custom-domain",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")},
},
namespace: "foo",
want: "department.org.io",
},
{
name: "success-default-domain",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.cluster.local."), toFQDN(t, "svc.cluster.local."), toFQDN(t, "cluster.local.")},
},
namespace: "foo",
want: "cluster.local",
},
{
name: "only-two-search-domains",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")},
},
namespace: "foo",
want: "cluster.local",
},
{
name: "first-search-domain-mismatch",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.bar.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")},
},
namespace: "foo",
want: "cluster.local",
},
{
name: "second-search-domain-mismatch",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "foo.department.org.io"), toFQDN(t, "some.other.fqdn")},
},
namespace: "foo",
want: "cluster.local",
},
{
name: "third-search-domain-mismatch",
conf: &resolvconffile.Config{
SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")},
},
namespace: "foo",
want: "cluster.local",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := clusterDomainFromResolverConf(tt.conf, tt.namespace, zl.Sugar()); got != tt.want {
t.Errorf("clusterDomainFromResolverConf() = %v, want %v", got, tt.want)
}
})
}
}
func Test_authKeyRemoval(t *testing.T) {
fc := fake.NewFakeClient()
ft := &fakeTSClient{}
@@ -2095,15 +2028,6 @@ func TestIgnorePGService(t *testing.T) {
findNoGenName(t, fc, "default", "test", "svc")
}
func toFQDN(t *testing.T, s string) dnsname.FQDN {
t.Helper()
fqdn, err := dnsname.ToFQDN(s)
if err != nil {
t.Fatalf("error coverting %q to dnsname.FQDN: %v", s, err)
}
return fqdn
}
func proxyCreatedCondition(clock tstime.Clock) []metav1.Condition {
return []metav1.Condition{{
Type: string(tsapi.ProxyReady),
cmd/k8s-operator/proxygroup.go
+14 -13
View File
@@ -34,6 +34,7 @@
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/egressservices"
@@ -205,7 +206,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, tsClient tsclien
if err != nil {
return r.notReadyErrf(pg, logger, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
if !reconciler.ProxyClassIsReady(proxyClass) {
msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName)
logger.Info(msg)
return notReady(reasonProxyGroupCreating, msg)
@@ -487,7 +488,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
pg.Status.Devices = devices
desiredReplicas := int(pgReplicas(pg))
desiredReplicas := int(reconciler.ProxyGroupReplicas(pg))
// Set ProxyGroupAvailable condition.
status := metav1.ConditionFalse
@@ -499,10 +500,10 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
reason = reasonProxyGroupAvailable
}
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger)
// Set ProxyGroupReady condition.
tsSvcValid, tsSvcSet := tsoperator.KubeAPIServerProxyValid(pg)
tsSvcValid, tsSvcSet := reconciler.KubeAPIServerProxyValid(pg)
status = metav1.ConditionFalse
reason = reasonProxyGroupCreating
switch {
@@ -516,7 +517,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
case len(devices) < desiredReplicas:
case len(devices) > desiredReplicas:
message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(devices)-desiredReplicas)
case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !tsoperator.KubeAPIServerProxyConfigured(pg):
case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !reconciler.KubeAPIServerProxyConfigured(pg):
reason = reasonProxyGroupCreating
message = "waiting for proxies to start advertising the kube-apiserver proxy's hostname"
default:
@@ -524,7 +525,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
reason = reasonProxyGroupReady
message = reasonProxyGroupReady
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger)
return nil
}
@@ -566,14 +567,14 @@ func (e *allocatePortsErr) Error() string {
}
func (r *ProxyGroupReconciler) allocatePorts(ctx context.Context, pg *tsapi.ProxyGroup, proxyClassName string, portRanges tsapi.PortRanges) (map[string]uint16, error) {
replicaCount := int(pgReplicas(pg))
replicaCount := int(reconciler.ProxyGroupReplicas(pg))
svcToNodePorts, usedPorts, err := getServicePortsForProxyGroups(ctx, r.Client, r.tsNamespace, portRanges)
if err != nil {
return nil, &allocatePortsErr{msg: fmt.Sprintf("failed to find ports for existing ProxyGroup NodePort Services: %s", err.Error())}
}
replicasAllocated := 0
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
if _, ok := svcToNodePorts[pgNodePortServiceName(pg.Name, i)]; !ok {
svcToNodePorts[pgNodePortServiceName(pg.Name, i)] = 0
} else {
@@ -605,7 +606,7 @@ func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context,
// NOTE: (ChaosInTheCRD) we want the same TargetPort for every static endpoint NodePort Service for the ProxyGroup
tailscaledPort := getRandomPort()
svcs := []*corev1.Service{}
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
nodePortSvcName := pgNodePortServiceName(pg.Name, i)
svc := &corev1.Service{}
@@ -663,7 +664,7 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, tsC
}
for _, m := range metadata {
if m.ordinal+1 <= pgReplicas(pg) {
if m.ordinal+1 <= reconciler.ProxyGroupReplicas(pg) {
continue
}
@@ -750,8 +751,8 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(
svcToNodePorts map[string]uint16,
) (endpoints map[string][]netip.AddrPort, err error) {
logger := r.logger(pg.Name)
endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) // keyed by Service name.
for i := range pgReplicas(pg) {
endpoints = make(map[string][]netip.AddrPort, reconciler.ProxyGroupReplicas(pg)) // keyed by Service name.
for i := range reconciler.ProxyGroupReplicas(pg) {
logger = logger.With("Pod", fmt.Sprintf("%s-%d", pg.Name, i))
cfgSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
@@ -1116,7 +1117,7 @@ func (r *ProxyGroupReconciler) ensureStateAddedForProxyGroup(pg *tsapi.ProxyGrou
gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len()))
r.reissuer.EnsureState(pg.Name, int(pgReplicas(pg)))
r.reissuer.EnsureState(pg.Name, int(reconciler.ProxyGroupReplicas(pg)))
}
// ensureStateRemovedForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the
cmd/k8s-operator/proxygroup_specs.go
+12 -22
View File
@@ -21,6 +21,8 @@
"sigs.k8s.io/yaml"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/egress"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/ingressservices"
"tailscale.com/kube/kubetypes"
@@ -92,7 +94,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
Labels: pgLabels(pg.Name, nil),
OwnerReferences: pgOwnerReference(pg),
}
ss.Spec.Replicas = new(pgReplicas(pg))
ss.Spec.Replicas = new(reconciler.ProxyGroupReplicas(pg))
ss.Spec.Selector = &metav1.LabelSelector{
MatchLabels: pgLabels(pg.Name, nil),
}
@@ -107,13 +109,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
}
tmpl.Spec.ServiceAccountName = pg.Name
tmpl.Spec.InitContainers[0].Image = image
proxyConfigVolName := pgEgressCMName(pg.Name)
proxyConfigVolName := egress.CMName(pg.Name)
if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
proxyConfigVolName = pgIngressCMName(pg.Name)
}
tmpl.Spec.Volumes = func() []corev1.Volume {
var volumes []corev1.Volume
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
volumes = append(volumes, corev1.Volume{
Name: fmt.Sprintf("tailscaledconfig-%d", i),
VolumeSource: corev1.VolumeSource{
@@ -147,7 +149,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
// TODO(tomhjp): Read config directly from the secret instead. The
// mounts change on scaling up/down which causes unnecessary restarts
// for pods that haven't meaningfully changed.
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
mounts = append(mounts, corev1.VolumeMount{
Name: fmt.Sprintf("tailscaledconfig-%d", i),
ReadOnly: true,
@@ -313,7 +315,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
// cluster egress traffic. The reconciler sets the corresponding
// condition; see egress-pod-readiness.go.
tmpl.Spec.ReadinessGates = append(tmpl.Spec.ReadinessGates, corev1.PodReadinessGate{
ConditionType: tsEgressReadinessGate,
ConditionType: egress.ReadinessGate,
})
}
@@ -329,7 +331,7 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
OwnerReferences: pgOwnerReference(pg),
},
Spec: appsv1.StatefulSetSpec{
Replicas: new(pgReplicas(pg)),
Replicas: new(reconciler.ProxyGroupReplicas(pg)),
Selector: &metav1.LabelSelector{
MatchLabels: pgLabels(pg.Name, nil),
},
@@ -464,7 +466,7 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string, shareACMEAccount bool) *rbac
"update",
},
ResourceNames: func() (secrets []string) {
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
secrets = append(secrets,
pgConfigSecretName(pg.Name, i), // Config with auth key.
pgPodName(pg.Name, i), // State.
@@ -565,7 +567,7 @@ func isAuthAPIServerProxy(pg *tsapi.ProxyGroup) bool {
}
func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) {
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
secrets = append(secrets, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgStateSecretName(pg.Name, i),
@@ -584,7 +586,7 @@ func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) (*corev1.ConfigMap, []by
hpBs := []byte(strconv.Itoa(hp))
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: pgEgressCMName(pg.Name),
Name: egress.CMName(pg.Name),
Namespace: namespace,
Labels: pgLabels(pg.Name, nil),
OwnerReferences: pgOwnerReference(pg),
@@ -625,14 +627,6 @@ func pgOwnerReference(owner *tsapi.ProxyGroup) []metav1.OwnerReference {
return []metav1.OwnerReference{*metav1.NewControllerRef(owner, tsapi.SchemeGroupVersion.WithKind("ProxyGroup"))}
}
func pgReplicas(pg *tsapi.ProxyGroup) int32 {
if pg.Spec.Replicas != nil {
return *pg.Spec.Replicas
}
return 2
}
func pgPodName(pgName string, i int32) string {
return fmt.Sprintf("%s-%d", pgName, i)
}
@@ -653,10 +647,6 @@ func pgStateSecretName(pgName string, i int32) string {
return fmt.Sprintf("%s-%d", pgName, i)
}
func pgEgressCMName(pg string) string {
return fmt.Sprintf("%s-egress-config", pg)
}
// hasLocalAddrPortSet returns true if the proxyclass has the TS_LOCAL_ADDR_PORT env var set. For egress ProxyGroups,
// currently (2025-01-26) this means that the ProxyGroup does not support graceful failover.
func hasLocalAddrPortSet(proxyClass *tsapi.ProxyClass) bool {
@@ -671,7 +661,7 @@ func hasLocalAddrPortSet(proxyClass *tsapi.ProxyClass) bool {
// hepPings returns the number of times a health check endpoint exposed by a Service fronting ProxyGroup replicas should
// be pinged to ensure that all currently configured backend replicas are hit.
func hepPings(pg *tsapi.ProxyGroup) int {
rc := pgReplicas(pg)
rc := reconciler.ProxyGroupReplicas(pg)
// Assuming a Service implemented using round robin load balancing, number-of-replica-times should be enough, but in
// practice, we cannot assume that the requests will be load balanced perfectly.
return int(rc) * 3
cmd/k8s-operator/proxygroup_test.go
+74 -72
View File
@@ -32,6 +32,8 @@
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/egress"
"tailscale.com/k8s-operator/reconciler/proxyclass"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
@@ -633,7 +635,7 @@ type reconcile struct {
WithScheme(tsapi.GlobalScheme).
Build()
reconciler := &ProxyGroupReconciler{
pgr := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
@@ -673,7 +675,7 @@ type reconcile struct {
t.Logf("created node %q with data", n.name)
}
reconciler.log = zl.Sugar().With("TestName", tt.name).With("Reconcile", i)
pgr.log = zl.Sugar().With("TestName", tt.name).With("Reconcile", i)
pg.Spec.Replicas = r.replicas
pc.Spec.StaticEndpoints = r.staticEndpointConfig
@@ -686,9 +688,9 @@ type reconcile struct {
})
if r.expectedErr != "" {
expectError(t, reconciler, "", pg.Name)
expectError(t, pgr, "", pg.Name)
} else {
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
}
expectEvents(t, fr, r.expectedEvents)
@@ -777,7 +779,7 @@ type reconcile struct {
}
t.Run("delete_and_cleanup", func(t *testing.T) {
reconciler := &ProxyGroupReconciler{
pgr := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
@@ -796,7 +798,7 @@ type reconcile struct {
t.Fatalf("error deleting ProxyGroup: %v", err)
}
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name)
if err := fc.Delete(t.Context(), pc); err != nil {
@@ -923,7 +925,7 @@ func TestProxyGroup(t *testing.T) {
zl, _ := zap.NewDevelopment()
fr := record.NewFakeRecorder(1)
cl := tstest.NewClock(tstest.ClockOpts{})
reconciler := &ProxyGroupReconciler{
pgr := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
@@ -948,13 +950,13 @@ func TestProxyGroup(t *testing.T) {
}
t.Run("proxyclass_not_ready", func(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, false, pc)
if tsoperator.ProxyGroupAvailable(pg) {
if reconciler.ProxyGroupAvailable(pg) {
t.Fatal("expected ProxyGroup to not be available")
}
})
@@ -976,17 +978,17 @@ func TestProxyGroup(t *testing.T) {
mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
p.ObjectMeta.Generation = pg.ObjectMeta.Generation
})
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 2, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 2, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc)
if tsoperator.ProxyGroupAvailable(pg) {
if reconciler.ProxyGroupAvailable(pg) {
t.Fatal("expected ProxyGroup to not be available")
}
if expected := 1; reconciler.egressProxyGroups.Len() != expected {
t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
if expected := 1; pgr.egressProxyGroups.Len() != expected {
t.Fatalf("expected %d egress ProxyGroups, got %d", expected, pgr.egressProxyGroups.Len())
}
expectProxyGroupResources(t, fc, pg, true, pc)
var keyReq tailscale.KeyCapabilities
@@ -1006,7 +1008,7 @@ func TestProxyGroup(t *testing.T) {
mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
p.ObjectMeta.Generation = pg.ObjectMeta.Generation
})
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
pg.Status.Devices = []tsapi.TailnetDevice{
{
@@ -1018,11 +1020,11 @@ func TestProxyGroup(t *testing.T) {
TailnetIPs: []string{"1.2.3.4", "::1"},
},
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc)
if !tsoperator.ProxyGroupAvailable(pg) {
if !reconciler.ProxyGroupAvailable(pg) {
t.Fatal("expected ProxyGroup to be available")
}
})
@@ -1032,16 +1034,16 @@ func TestProxyGroup(t *testing.T) {
mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
p.Spec = pg.Spec
})
expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 3, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
expectReconciled(t, pgr, "", pg.Name)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 3, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc)
addNodeIDToStateSecrets(t, fc, pg)
expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar())
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar())
expectReconciled(t, pgr, "", pg.Name)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar())
pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{
Hostname: "hostname-nodeid-2",
TailnetIPs: []string{"1.2.3.4", "::1"},
@@ -1056,10 +1058,10 @@ func TestProxyGroup(t *testing.T) {
p.Spec = pg.Spec
})
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar())
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc)
})
@@ -1069,7 +1071,7 @@ func TestProxyGroup(t *testing.T) {
mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
p.Spec = pc.Spec
})
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
expectEqual(t, fc, expectedMetricsService(opts))
})
t.Run("enable_service_monitor_no_crd", func(t *testing.T) {
@@ -1077,11 +1079,11 @@ func TestProxyGroup(t *testing.T) {
mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
p.Spec.Metrics = pc.Spec.Metrics
})
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
})
t.Run("create_crd_expect_service_monitor", func(t *testing.T) {
mustCreate(t, fc, crd)
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
})
@@ -1090,18 +1092,18 @@ func TestProxyGroup(t *testing.T) {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name)
if expected := 0; reconciler.egressProxyGroups.Len() != expected {
t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
if expected := 0; pgr.egressProxyGroups.Len() != expected {
t.Fatalf("expected %d ProxyGroups, got %d", expected, pgr.egressProxyGroups.Len())
}
// 2 nodes should get deleted as part of the scale down, and then finally
// the first node gets deleted with the ProxyGroup cleanup.
if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-1", "nodeid-2", "nodeid-0"}); diff != "" {
t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
}
expectMissing[corev1.Service](t, reconciler, "tailscale", metricsResourceName(pg.Name))
expectMissing[corev1.Service](t, pgr, "tailscale", metricsResourceName(pg.Name))
// The fake client does not clean up objects whose owner has been
// deleted, so we can't test for the owned resources getting deleted.
})
@@ -1131,7 +1133,7 @@ func TestProxyGroupTypes(t *testing.T) {
})
zl, _ := zap.NewDevelopment()
reconciler := &ProxyGroupReconciler{
pgr := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
@@ -1155,8 +1157,8 @@ func TestProxyGroupTypes(t *testing.T) {
}
mustCreate(t, fc, pg)
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 0, 1, 0)
expectReconciled(t, pgr, "", pg.Name)
verifyProxyGroupCounts(t, pgr, 0, 1, 0)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@@ -1217,9 +1219,9 @@ func TestProxyGroupTypes(t *testing.T) {
t.Errorf("unexpected deletion grace period seconds %d, want %d", *sts.Spec.Template.DeletionGracePeriodSeconds, deletionGracePeriodSeconds)
}
if !slices.ContainsFunc(sts.Spec.Template.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool {
return r.ConditionType == tsEgressReadinessGate
return r.ConditionType == egress.ReadinessGate
}) {
t.Errorf("expected egress readiness gate %q to be set, got %v", tsEgressReadinessGate, sts.Spec.Template.Spec.ReadinessGates)
t.Errorf("expected egress readiness gate %q to be set, got %v", egress.ReadinessGate, sts.Spec.Template.Spec.ReadinessGates)
}
})
t.Run("egress_type_no_lifecycle_hook_when_local_addr_port_set", func(t *testing.T) {
@@ -1247,7 +1249,7 @@ func TestProxyGroupTypes(t *testing.T) {
},
}
})
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@@ -1258,7 +1260,7 @@ func TestProxyGroupTypes(t *testing.T) {
t.Error("lifecycle hook was set when TS_LOCAL_ADDR_PORT was configured via ProxyClass")
}
if slices.ContainsFunc(sts.Spec.Template.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool {
return r.ConditionType == tsEgressReadinessGate
return r.ConditionType == egress.ReadinessGate
}) {
t.Error("egress readiness gate was set when TS_LOCAL_ADDR_PORT was configured via ProxyClass")
}
@@ -1282,8 +1284,8 @@ func TestProxyGroupTypes(t *testing.T) {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 1, 2, 0)
expectReconciled(t, pgr, "", pg.Name)
verifyProxyGroupCounts(t, pgr, 1, 2, 0)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@@ -1381,7 +1383,7 @@ func TestProxyGroupTypes(t *testing.T) {
if err := fc.Create(t.Context(), pg); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@@ -1431,8 +1433,8 @@ func TestProxyGroupTypes(t *testing.T) {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 2, 2, 1)
expectReconciled(t, pgr, "", pg.Name)
verifyProxyGroupCounts(t, pgr, 2, 2, 1)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@@ -1490,17 +1492,17 @@ func TestKubeAPIServerStatusConditionFlow(t *testing.T) {
expectReconciled(t, r, "", pg.Name)
pg.ObjectMeta.Finalizers = append(pg.ObjectMeta.Finalizers, FinalizerName)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.log)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.log)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set kube-apiserver valid.
mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log)
reconciler.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log)
})
expectReconciled(t, r, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set available.
@@ -1512,17 +1514,17 @@ func TestKubeAPIServerStatusConditionFlow(t *testing.T) {
TailnetIPs: []string{"1.2.3.4", "::1"},
},
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.log)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.log)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set kube-apiserver configured.
mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log)
reconciler.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log)
})
expectReconciled(t, r, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.log)
reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.log)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
}
@@ -1532,7 +1534,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) {
WithStatusSubresource(&tsapi.ProxyGroup{}).
Build()
reconciler := &ProxyGroupReconciler{
pgr := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
@@ -1558,7 +1560,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) {
if err := fc.Create(t.Context(), pg); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
cfg := conf.VersionedConfig{
Version: "v1alpha1",
@@ -1607,7 +1609,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) {
mustUpdate(t, fc, tsNamespace, cfgSecret.Name, func(s *corev1.Secret) {
s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB
})
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
cfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cfgB
expectEqual(t, fc, cfgSecret)
@@ -1618,7 +1620,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
WithScheme(tsapi.GlobalScheme).
WithStatusSubresource(&tsapi.ProxyGroup{}).
Build()
reconciler := &ProxyGroupReconciler{
pgr := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
@@ -1658,7 +1660,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
Replicas: new(int32(1)),
},
})
expectReconciled(t, reconciler, "", pgName)
expectReconciled(t, pgr, "", pgName)
expectedConfigBytes, err := json.Marshal(ipn.ConfigVAlpha{
// Preserved.
@@ -1885,7 +1887,7 @@ func TestProxyGroupGetAuthKey(t *testing.T) {
zl, _ := zap.NewDevelopment()
fr := record.NewFakeRecorder(1)
cl := tstest.NewClock(tstest.ClockOpts{})
reconciler := &ProxyGroupReconciler{
pgr := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
@@ -1898,9 +1900,9 @@ func TestProxyGroupGetAuthKey(t *testing.T) {
clock: cl,
reissuer: tailscaled.NewReissuer(),
}
reconciler.ensureStateAddedForProxyGroup(pg)
pgr.ensureStateAddedForProxyGroup(pg)
return reconciler, fc
return pgr, fc
}
// Config Secret: exists or not, has key or not.
@@ -1970,7 +1972,7 @@ func TestProxyGroupGetAuthKey(t *testing.T) {
} {
t.Run(name, func(t *testing.T) {
tsClient.deleted = tsClient.deleted[:0] // Reset deleted devices for each test case.
reconciler, fc := initTest()
pgr, fc := initTest()
var cfgSecret *corev1.Secret
if tc.configData != nil {
cfgSecret = &corev1.Secret{
@@ -1991,7 +1993,7 @@ func TestProxyGroupGetAuthKey(t *testing.T) {
})
}
authKey, err := reconciler.getAuthKey(t.Context(), tsClient, pg, cfgSecret, 0, reconciler.log.With("TestName", t.Name()))
authKey, err := pgr.getAuthKey(t.Context(), tsClient, pg, cfgSecret, 0, pgr.log.With("TestName", t.Name()))
if err != nil {
t.Fatalf("unexpected error getting auth key: %v", err)
}
@@ -2135,7 +2137,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
var expectedSecrets []string
if shouldExist {
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
expectedSecrets = append(expectedSecrets,
fmt.Sprintf("%s-%d", pg.Name, i),
pgConfigSecretName(pg.Name, i),
@@ -2166,7 +2168,7 @@ func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) {
func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) {
t.Helper()
const key = "profile-abc"
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
bytes, err := json.Marshal(map[string]any{
"Config": map[string]any{
"NodeID": fmt.Sprintf("nodeid-%d", i),
@@ -2260,7 +2262,7 @@ type proxyGroupLETestCase struct {
setProxyClassReady(t, fc, cl, name)
}
reconciler := &ProxyGroupReconciler{
pgr := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test"},
@@ -2272,7 +2274,7 @@ type proxyGroupLETestCase struct {
reissuer: tailscaled.NewReissuer(),
}
expectReconciled(t, reconciler, "", pg.Name)
expectReconciled(t, pgr, "", pg.Name)
// Verify that the StatefulSet created for ProxyGrup has
// the expected setting for the staging endpoint.
cmd/k8s-operator/sts.go
+7 -8
View File
@@ -34,6 +34,7 @@
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/net/netutil"
@@ -68,6 +69,11 @@
AnnotationProxyGroup = "tailscale.com/proxy-group"
// labelProxyGroup names the ProxyGroup a managed resource belongs to. Same string as
// AnnotationProxyGroup, which users set on the Service; this is the label the operator
// stamps on the resources it creates in response.
labelProxyGroup = "tailscale.com/proxy-group"
// AnnotationShareACMEAccount opts a single ProxyGroup into ("true")
// or out of ("false") using the shared per-tailnet ACME account key.
// When absent, OPERATOR_SHARED_ACME_ACCOUNT_KEY on the operator is
@@ -216,7 +222,7 @@ func (r *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
if err := r.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil {
return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
if !reconciler.ProxyClassIsReady(proxyClass) {
logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
return nil, nil
}
@@ -1288,13 +1294,6 @@ func defaultEnv(envName, defVal string) string {
return v
}
func nameForService(svc *corev1.Service) string {
if h, ok := svc.Annotations[AnnotationHostname]; ok {
return h
}
return svc.Namespace + "-" + svc.Name
}
// markedForDeletion reports whether obj has a deletion timestamp, i.e. the API server is waiting on finalizers
// before it can garbage collect it.
func markedForDeletion(obj metav1.Object) bool {
cmd/k8s-operator/svc-for-pg.go
+11 -11
View File
@@ -29,8 +29,8 @@
"tailscale.com/client/tailscale/v2"
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/k8s-operator/tsclient"
"tailscale.com/kube/ingressservices"
@@ -116,7 +116,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque
return res, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err)
}
if !tsoperator.ProxyGroupAvailable(pg) {
if !reconciler.ProxyGroupAvailable(pg) {
logger.Infof("ProxyGroup is not (yet) ready")
return res, nil
}
@@ -126,7 +126,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque
return res, fmt.Errorf("failed to get tailscale client: %w", err)
}
hostname := nameForService(svc)
hostname := reconciler.NameForService(svc)
logger = logger.With("hostname", hostname)
if !svc.DeletionTimestamp.IsZero() || !r.isTailscaleService(svc) {
@@ -173,7 +173,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
if err = r.validateService(ctx, svc, pg); err != nil {
r.recorder.Event(svc, corev1.EventTypeWarning, reasonIngressSvcInvalid, err.Error())
tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, err.Error(), r.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, err.Error(), r.clock, logger)
return false, nil
}
@@ -224,7 +224,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr)
logger.Warn(msg)
r.recorder.Event(svc, corev1.EventTypeWarning, "InvalidTailscaleService", msg)
tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, msg, r.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, msg, r.clock, logger)
return false, nil
}
@@ -347,7 +347,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
conditionStatus := metav1.ConditionFalse
conditionType := tsapi.IngressSvcConfigured
conditionReason := reasonIngressSvcNoBackendsConfigured
conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", count, pgReplicas(pg))
conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", count, reconciler.ProxyGroupReplicas(pg))
if count != 0 {
dnsName, err := dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace)
if err != nil {
@@ -365,7 +365,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
conditionReason = reasonIngressSvcConfigured
}
tsoperator.SetServiceCondition(svc, conditionType, conditionStatus, conditionReason, conditionMessage, r.clock, logger)
reconciler.SetServiceCondition(svc, conditionType, conditionStatus, conditionReason, conditionMessage, r.clock, logger)
svc.Status.LoadBalancer.Ingress = lbs
return svcsChanged, nil
@@ -441,7 +441,7 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
for tsSvcName, cfg := range config {
found := false
for _, svc := range svcList.Items {
if strings.EqualFold(fmt.Sprintf("svc:%s", nameForService(&svc)), tsSvcName) {
if strings.EqualFold(fmt.Sprintf("svc:%s", reconciler.NameForService(&svc)), tsSvcName) {
found = true
break
}
@@ -820,7 +820,7 @@ func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.S
errs = append(errs, fmt.Errorf("ProxyGroup %q is of type %q but must be of type %q",
pg.Name, pg.Spec.Type, tsapi.ProxyGroupTypeIngress))
}
if violations := validateService(svc); len(violations) > 0 {
if violations := reconciler.ValidateService(svc); len(violations) > 0 {
errs = append(errs, fmt.Errorf("invalid Service: %s", strings.Join(violations, ", ")))
}
svcList := &corev1.ServiceList{}
@@ -828,7 +828,7 @@ func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.S
errs = append(errs, fmt.Errorf("error listing Services: %w", err))
return errors.Join(errs...)
}
svcName := nameForService(svc)
svcName := reconciler.NameForService(svc)
for _, s := range svcList.Items {
if s.UID == svc.UID {
continue
@@ -842,7 +842,7 @@ func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.S
if !r.isTailscaleService(&s) {
continue
}
if nameForService(&s) != svcName {
if reconciler.NameForService(&s) != svcName {
continue
}
// Two ProxyGroups joined to different tailnets each have their own
cmd/k8s-operator/svc.go
+19 -106
View File
@@ -24,20 +24,15 @@
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
"tailscale.com/net/dns/resolvconffile"
"tailscale.com/tstime"
"tailscale.com/util/clientmetric"
"tailscale.com/util/dnsname"
"tailscale.com/util/set"
)
const (
resolvConfPath = "/etc/resolv.conf"
defaultClusterDomain = "cluster.local"
reasonProxyCreated = "ProxyCreated"
reasonProxyInvalid = "ProxyInvalid"
reasonProxyFailed = "ProxyFailed"
@@ -156,7 +151,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
if !a.isTailscaleService(svc) {
tsoperator.RemoveServiceCondition(svc, tsapi.ProxyReady)
reconciler.RemoveServiceCondition(svc, tsapi.ProxyReady)
}
return nil
}
@@ -192,7 +187,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
if !a.isTailscaleService(svc) {
tsoperator.RemoveServiceCondition(svc, tsapi.ProxyReady)
reconciler.RemoveServiceCondition(svc, tsapi.ProxyReady)
}
return nil
}
@@ -218,14 +213,14 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
msg := fmt.Sprintf("unable to provision proxy resources: invalid config: %v", err)
a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDCONFIG", msg)
a.logger.Error(msg)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger)
return nil
}
if violations := validateService(svc); len(violations) > 0 {
if violations := reconciler.ValidateService(svc); len(violations) > 0 {
msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", "))
a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
a.logger.Error(msg)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger)
return nil
}
@@ -233,11 +228,11 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
if proxyClass != "" {
if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
return errMsg
} else if !ready {
msg := fmt.Sprintf("ProxyClass %s specified for the Service, but is not (yet) Ready, waiting..", proxyClass)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger)
logger.Info(msg)
return nil
}
@@ -252,7 +247,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
svc.Finalizers = append(svc.Finalizers, FinalizerName)
if err := a.Update(ctx, svc); err != nil {
errMsg := fmt.Errorf("failed to add finalizer: %w", err)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
return errMsg
}
}
@@ -266,7 +261,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
Replicas: 1,
ParentResourceName: svc.Name,
ParentResourceUID: string(svc.UID),
Hostname: nameForService(svc),
Hostname: reconciler.NameForService(svc),
Tags: tags,
ChildResourceLabels: crl,
ProxyClassName: proxyClass,
@@ -304,12 +299,12 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
var hsvc *corev1.Service
if hsvc, err = a.ssr.Provision(ctx, logger, sts); err != nil {
errMsg := fmt.Errorf("failed to provision: %w", err)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
return errMsg
}
if sts.TailnetTargetIP != "" || sts.TailnetTargetFQDN != "" { // if an egress proxy
clusterDomain := retrieveClusterDomain(a.tsNamespace, logger)
clusterDomain := reconciler.ClusterDomain(a.tsNamespace, logger)
headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc." + clusterDomain
if svc.Spec.ExternalName != headlessSvcName || svc.Spec.Type != corev1.ServiceTypeExternalName {
svc.Spec.ExternalName = headlessSvcName
@@ -317,17 +312,17 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
svc.Spec.Type = corev1.ServiceTypeExternalName
if err := a.Update(ctx, svc); err != nil {
errMsg := fmt.Errorf("failed to update service: %w", err)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
return errMsg
}
}
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
return nil
}
if !isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) {
logger.Debugf("service is not a LoadBalancer, so not updating ingress")
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
return nil
}
@@ -341,7 +336,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
logger.Debug(msg)
// No hostname yet. Wait for the proxy pod to auth.
svc.Status.LoadBalancer.Ingress = nil
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger)
return nil
}
@@ -355,7 +350,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
if err != nil {
msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
return errors.New(msg)
}
@@ -370,44 +365,10 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
}
svc.Status.LoadBalancer.Ingress = ingress
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
return nil
}
func validateService(svc *corev1.Service) []string {
violations := make([]string, 0)
if svc.Spec.ClusterIP == "None" {
violations = append(violations, "headless Services are not supported.")
}
if svc.Annotations[AnnotationTailnetTargetFQDN] != "" && svc.Annotations[AnnotationTailnetTargetIP] != "" {
violations = append(violations, fmt.Sprintf("only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN))
}
if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
if !isMagicDNSName(fqdn) {
violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
}
}
if ipStr := svc.Annotations[AnnotationTailnetTargetIP]; ipStr != "" {
ip, err := netip.ParseAddr(ipStr)
if err != nil {
violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q could not be parsed as a valid IP Address, error: %s", AnnotationTailnetTargetIP, ipStr, err))
} else if !ip.IsValid() {
violations = append(violations, fmt.Sprintf("parsed IP address in annotation %s: %q is not valid", AnnotationTailnetTargetIP, ipStr))
}
}
svcName := nameForService(svc)
if err := dnsname.ValidLabel(svcName); err != nil {
if _, ok := svc.Annotations[AnnotationHostname]; ok {
violations = append(violations, fmt.Sprintf("invalid Tailscale hostname specified %q: %s", svcName, err))
} else {
violations = append(violations, fmt.Sprintf("invalid Tailscale hostname %q, use %q annotation to override: %s", svcName, AnnotationHostname, err))
}
}
violations = append(violations, tagViolations(svc)...)
return violations
}
func shouldExpose(svc *corev1.Service, isDefaultLoadBalancer bool) bool {
return shouldExposeClusterIP(svc, isDefaultLoadBalancer) || shouldExposeDNSName(svc)
}
@@ -455,53 +416,5 @@ func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool
if err := cl.Get(ctx, types.NamespacedName{Name: name}, proxyClass); err != nil {
return false, fmt.Errorf("error getting ProxyClass %s: %w", name, err)
}
return tsoperator.ProxyClassIsReady(proxyClass), nil
}
// retrieveClusterDomain determines and retrieves cluster domain i.e
// (cluster.local) in which this Pod is running by parsing search domains in
// /etc/resolv.conf. If an error is encountered at any point during the process,
// defaults cluster domain to 'cluster.local'.
func retrieveClusterDomain(namespace string, logger *zap.SugaredLogger) string {
logger.Infof("attempting to retrieve cluster domain..")
conf, err := resolvconffile.ParseFile(resolvConfPath)
if err != nil {
// Vast majority of clusters use the cluster.local domain, so it
// is probably better to fall back to that than error out.
logger.Warn("error parsing /etc/resolv.conf to determine cluster domain, defaulting to 'cluster.local'.")
return defaultClusterDomain
}
return clusterDomainFromResolverConf(conf, namespace, logger)
}
// clusterDomainFromResolverConf attempts to retrieve cluster domain from the provided resolver config.
// It expects the first three search domains in the resolver config to be ['<namespace>.svc.<cluster-domain>, svc.<cluster-domain>, <cluster-domain>, ...]
// If the first three domains match the expected structure, it returns the third.
// If the domains don't match the expected structure or an error is encountered, it defaults to 'cluster.local' domain.
func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string, logger *zap.SugaredLogger) string {
if len(conf.SearchDomains) < 3 {
logger.Warnf(" resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.", len(conf.SearchDomains))
return defaultClusterDomain
}
first := conf.SearchDomains[0]
if !strings.HasPrefix(string(first), namespace+".svc") {
logger.Warnf("first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.<cluster-domain>")
return defaultClusterDomain
}
second := conf.SearchDomains[1]
if !strings.HasPrefix(string(second), "svc") {
logger.Warnf("second search domain in resolver config is %s; expected 'svc.<cluster-domain>'.\nDefaulting cluster domain to 'cluster.local'.", second)
return defaultClusterDomain
}
// Trim the trailing dot for backwards compatibility purposes as the
// cluster domain was previously hardcoded to 'cluster.local' without a
// trailing dot.
probablyClusterDomain := strings.TrimPrefix(second.WithoutTrailingDot(), "svc.")
third := conf.SearchDomains[2]
if !strings.EqualFold(third.WithoutTrailingDot(), probablyClusterDomain) {
logger.Warnf("expected resolver config to contain serch domains <namespace>.svc.<cluster-domain>, svc.<cluster-domain>, <cluster-domain>; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third)
return defaultClusterDomain
}
logger.Infof("Cluster domain %q extracted from resolver config", probablyClusterDomain)
return probablyClusterDomain
return reconciler.ProxyClassIsReady(proxyClass), nil
}
cmd/k8s-operator/testutils_test.go
-28
View File
@@ -664,21 +664,6 @@ func mustCreate(t *testing.T, client client.Client, obj client.Object) {
t.Fatalf("creating %q: %v", obj.GetName(), err)
}
}
func mustCreateAll(t *testing.T, client client.Client, objs ...client.Object) {
t.Helper()
for _, obj := range objs {
mustCreate(t, client, obj)
}
}
func mustDeleteAll(t *testing.T, client client.Client, objs ...client.Object) {
t.Helper()
for _, obj := range objs {
if err := client.Delete(context.Background(), obj); err != nil {
t.Fatalf("deleting %q: %v", obj.GetName(), err)
}
}
}
func mustUpdate[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string, update func(O)) {
t.Helper()
@@ -985,19 +970,6 @@ func removeResourceReqs(sts *appsv1.StatefulSet) {
}
}
func removeTargetPortsFromSvc(svc *corev1.Service) {
newPorts := make([]corev1.ServicePort, 0)
for _, p := range svc.Spec.Ports {
newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port, Name: p.Name})
}
svc.Spec.Ports = newPorts
}
func removeClusterIPsFromSvc(svc *corev1.Service) {
svc.Spec.ClusterIP = ""
svc.Spec.ClusterIPs = nil
}
func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
return func(secret *corev1.Secret) {
t.Helper()
k8s-operator/conditions.go
-217
View File
@@ -1,217 +0,0 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package kube
import (
"slices"
"time"
"go.uber.org/zap"
xslices "golang.org/x/exp/slices"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tstime"
)
// SetConnectorCondition ensures that Connector status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(cn.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
cn.Status.Conditions = conds
}
// RemoveConnectorCondition will remove condition of the given type if it exists.
func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConditionType) {
conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(conditionType)
})
}
// SetProxyClassCondition ensures that ProxyClass status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetProxyClassCondition(pc *tsapi.ProxyClass, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(pc.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
pc.Status.Conditions = conds
}
// SetDNSConfigCondition ensures that DNSConfig status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes
func SetDNSConfigCondition(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(dnsCfg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
dnsCfg.Status.Conditions = conds
}
// SetServiceCondition ensures that Service status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(svc.Status.Conditions, conditionType, status, reason, message, 0, clock, logger)
svc.Status.Conditions = conds
}
// GetServiceCondition returns Service condition with the specified type, if it exists on the Service.
func GetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) *metav1.Condition {
idx := xslices.IndexFunc(svc.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(conditionType)
})
if idx == -1 {
return nil
}
return &svc.Status.Conditions[idx]
}
// RemoveServiceCondition will remove condition of the given type if it exists.
func RemoveServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) {
svc.Status.Conditions = slices.DeleteFunc(svc.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(conditionType)
})
}
// SetRecorderCondition ensures that Recorder status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetRecorderCondition(tsr *tsapi.Recorder, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(tsr.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
tsr.Status.Conditions = conds
}
// SetProxyGroupCondition ensures that ProxyGroup status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetProxyGroupCondition(pg *tsapi.ProxyGroup, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(pg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
pg.Status.Conditions = conds
}
// SetTailnetCondition ensures that Tailnet status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetTailnetCondition(tn *tsapi.Tailnet, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(tn.Status.Conditions, conditionType, status, reason, message, tn.Generation, clock, logger)
tn.Status.Conditions = conds
}
// SetPeerRelayCondition ensures that PeerRelay status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetPeerRelayCondition(pr *tsapi.PeerRelay, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) {
conds := updateCondition(pr.Status.Conditions, conditionType, status, reason, message, pr.Generation, clock, logger)
pr.Status.Conditions = conds
}
func updateCondition(conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []metav1.Condition {
newCondition := metav1.Condition{
Type: string(conditionType),
Status: status,
Reason: reason,
Message: message,
ObservedGeneration: gen,
}
nowTime := metav1.NewTime(clock.Now().Truncate(time.Second))
newCondition.LastTransitionTime = nowTime
idx := xslices.IndexFunc(conds, func(cond metav1.Condition) bool {
return cond.Type == string(conditionType)
})
if idx == -1 {
conds = append(conds, newCondition)
return conds
}
cond := conds[idx] // update the existing condition
// If this update doesn't contain a state transition, don't update last
// transition time.
if cond.Status == status {
newCondition.LastTransitionTime = cond.LastTransitionTime
} else {
logger.Infof("Status change for condition %s from %s to %s", conditionType, cond.Status, status)
}
conds[idx] = newCondition
return conds
}
func ProxyClassIsReady(pc *tsapi.ProxyClass) bool {
idx := xslices.IndexFunc(pc.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(tsapi.ProxyClassReady)
})
if idx == -1 {
return false
}
cond := pc.Status.Conditions[idx]
return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pc.Generation
}
func ProxyGroupIsReady(pg *tsapi.ProxyGroup) bool {
cond := proxyGroupCondition(pg, tsapi.ProxyGroupReady)
return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation
}
func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool {
cond := proxyGroupCondition(pg, tsapi.ProxyGroupAvailable)
return cond != nil && cond.Status == metav1.ConditionTrue
}
func KubeAPIServerProxyValid(pg *tsapi.ProxyGroup) (valid bool, set bool) {
cond := proxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid)
return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation, cond != nil
}
func KubeAPIServerProxyConfigured(pg *tsapi.ProxyGroup) bool {
cond := proxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured)
return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation
}
func proxyGroupCondition(pg *tsapi.ProxyGroup, condType tsapi.ConditionType) *metav1.Condition {
idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(condType)
})
if idx == -1 {
return nil
}
return &pg.Status.Conditions[idx]
}
func DNSCfgIsReady(cfg *tsapi.DNSConfig) bool {
idx := xslices.IndexFunc(cfg.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(tsapi.NameserverReady)
})
if idx == -1 {
return false
}
cond := cfg.Status.Conditions[idx]
return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == cfg.Generation
}
func SvcIsReady(svc *corev1.Service) bool {
idx := xslices.IndexFunc(svc.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(tsapi.ProxyReady)
})
if idx == -1 {
return false
}
cond := svc.Status.Conditions[idx]
return cond.Status == metav1.ConditionTrue
}
func TailnetIsReady(tn *tsapi.Tailnet) bool {
idx := xslices.IndexFunc(tn.Status.Conditions, func(cond metav1.Condition) bool {
return cond.Type == string(tsapi.TailnetReady)
})
if idx == -1 {
return false
}
cond := tn.Status.Conditions[idx]
return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == tn.Generation
}
k8s-operator/conditions_test.go
-101
View File
@@ -1,101 +0,0 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package kube
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tstest"
)
func TestSetConnectorCondition(t *testing.T) {
cn := tsapi.Connector{}
clock := tstest.NewClock(tstest.ClockOpts{})
fakeNow := metav1.NewTime(clock.Now().Truncate(time.Second))
fakePast := metav1.NewTime(clock.Now().Truncate(time.Second).Add(-5 * time.Minute))
zl, err := zap.NewDevelopment()
assert.Nil(t, err)
// Set up a new condition
SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "someReason", "someMsg", 1, clock, zl.Sugar())
assert.Equal(t, cn, tsapi.Connector{
Status: tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
Reason: "someReason",
Message: "someMsg",
ObservedGeneration: 1,
LastTransitionTime: fakeNow,
},
},
},
})
// Modify status of an existing condition
cn.Status = tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionFalse,
Reason: "someReason",
Message: "someMsg",
ObservedGeneration: 1,
LastTransitionTime: fakePast,
},
},
}
SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar())
assert.Equal(t, cn, tsapi.Connector{
Status: tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
Reason: "anotherReason",
Message: "anotherMsg",
ObservedGeneration: 2,
LastTransitionTime: fakeNow,
},
},
},
})
// Don't modify last transition time if status hasn't changed
cn.Status = tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
Reason: "someReason",
Message: "someMsg",
ObservedGeneration: 1,
LastTransitionTime: fakePast,
},
},
}
SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar())
assert.Equal(t, cn, tsapi.Connector{
Status: tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
Reason: "anotherReason",
Message: "anotherMsg",
ObservedGeneration: 2,
LastTransitionTime: fakePast,
},
},
},
})
}
k8s-operator/reconciler/dnsrecords/dnsrecords.go
+12 -7
View File
@@ -21,6 +21,7 @@
networkingv1 "k8s.io/api/networking/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/net"
"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -48,9 +49,6 @@
shortRequeue = 5 * time.Second
// AnnotationTailnetTargetFQDN is the annotation used to configure an egress proxy's tailnet target FQDN.
AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn"
labelProxyGroup = "tailscale.com/proxy-group"
labelSvcType = "tailscale.com/svc-type"
typeEgress = "egress"
@@ -150,7 +148,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
return reconcile.Result{}, nil
}
dnsCfg := dnsCfgLst.Items[0]
if !operatorutils.DNSCfgIsReady(&dnsCfg) {
if !nameserverReady(&dnsCfg) {
logger.Info("DNSConfig is not ready yet, waiting...")
return reconcile.Result{}, nil
}
@@ -343,7 +341,7 @@ func (r *Reconciler) fqdnForDNSRecord(ctx context.Context, proxySvc *corev1.Serv
return "", err
}
return svc.Annotations[AnnotationTailnetTargetFQDN], nil
return svc.Annotations[reconciler.AnnotationTailnetTargetFQDN], nil
}
return "", nil
}
@@ -391,7 +389,7 @@ func (r *Reconciler) isSvcForFQDNEgressProxy(ctx context.Context, svc *corev1.Se
return false, err
}
annots := parentSvc.Annotations
return annots != nil && annots[AnnotationTailnetTargetFQDN] != "", nil
return annots != nil && annots[reconciler.AnnotationTailnetTargetFQDN] != "", nil
}
// isProxyGroupEgressService reports whether the Service is a ClusterIP Service
@@ -435,7 +433,7 @@ func (r *Reconciler) parentSvcTargetsFQDN(ctx context.Context, svc *corev1.Servi
if err := r.Get(ctx, parentName, parentSvc); err != nil {
return false
}
return parentSvc.Annotations[AnnotationTailnetTargetFQDN] != ""
return parentSvc.Annotations[reconciler.AnnotationTailnetTargetFQDN] != ""
}
// getTargetIPs returns the IPv4 and IPv6 addresses that should be used for DNS records
@@ -621,3 +619,10 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
return reqs
}
}
// nameserverReady reports whether the DNSConfig's nameserver is ready for the config's current generation, i.e. there
// is an in-cluster ts.net nameserver to write records for.
func nameserverReady(cfg *tsapi.DNSConfig) bool {
cond := reconciler.Condition(cfg.Status.Conditions, tsapi.NameserverReady)
return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == cfg.Generation
}
k8s-operator/reconciler/dnsrecords/dnsrecords_test.go
+4 -4
View File
@@ -83,7 +83,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
cl := tstest.NewClock(tstest.ClockOpts{})
// Set the ready condition of the DNSConfig
reconcilertest.MustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) {
operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, nameserver.ReasonNameserverCreated, nameserver.ReasonNameserverCreated, 0, cl, zl.Sugar())
reconciler.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, nameserver.ReasonNameserverCreated, nameserver.ReasonNameserverCreated, 0, cl, zl.Sugar())
})
dnsRR := dnsrecords.NewReconciler(dnsrecords.ReconcilerOptions{
Client: fc,
@@ -183,7 +183,7 @@ func TestDNSRecordsReconciler(t *testing.T) {
Name: "external-service",
Namespace: "default",
Annotations: map[string]string{
dnsrecords.AnnotationTailnetTargetFQDN: "external-service.example.ts.net",
reconciler.AnnotationTailnetTargetFQDN: "external-service.example.ts.net",
},
},
Spec: corev1.ServiceSpec{
@@ -297,7 +297,7 @@ funcs := interceptor.Funcs{
Name: "lock-service",
Namespace: "default",
Annotations: map[string]string{
dnsrecords.AnnotationTailnetTargetFQDN: "lock-service.example.ts.net",
reconciler.AnnotationTailnetTargetFQDN: "lock-service.example.ts.net",
},
},
Spec: corev1.ServiceSpec{
@@ -433,7 +433,7 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) {
Name: "pg-service",
Namespace: "tailscale",
Annotations: map[string]string{
dnsrecords.AnnotationTailnetTargetFQDN: "pg-service.example.ts.net",
reconciler.AnnotationTailnetTargetFQDN: "pg-service.example.ts.net",
},
},
Spec: corev1.ServiceSpec{
k8s-operator/reconciler/egress/egress.go
+107
View File
@@ -0,0 +1,107 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package egress provides the reconcilers that expose tailnet targets to cluster workloads via an egress ProxyGroup.
//
// A user creates an ExternalName Service annotated with a tailnet target and a ProxyGroup. Four controllers then
// cooperate to make that target reachable:
//
// - Reconciler owns the user's ExternalName Service. It allocates a container port per Service port, creates a
// ClusterIP Service holding those portmappings, and writes the egress config the ProxyGroup's proxies read.
// - EndpointSliceReconciler keeps the ClusterIP Service's EndpointSlices pointing at the proxy Pods that are
// currently able to route traffic to the target.
// - ReadinessReconciler surfaces, on the user's Service, whether any proxy is actually ready to route to it.
// - PodReconciler gates proxy Pod readiness on the Pod having set up routing for its egress services, so that a
// rolling restart doesn't black-hole traffic.
//
// They are separate controllers rather than one because they watch different resources and must make progress
// independently; they share this package because they read and write the same egress config and labels.
package egress
import (
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
)
const (
// shortRequeue is how long to wait before retrying a reconcile that is waiting on state it doesn't control, e.g.
// a proxy Pod finishing its routing setup.
shortRequeue = 5 * time.Second
// labelProxyGroup names the ProxyGroup that a managed resource belongs to.
labelProxyGroup = "tailscale.com/proxy-group"
// labelSvcType distinguishes ingress from egress managed resources.
labelSvcType = "tailscale.com/svc-type"
typeEgress = "egress"
// parentTypeSvc is the LabelParentType value used for resources owned by a user-created Service.
parentTypeSvc = "svc"
// proxyTypeProxyGroup is the LabelParentType value carried by ProxyGroup-owned Pods.
proxyTypeProxyGroup = "proxygroup"
// tsHealthCheckPortName names the port on the ClusterIP Service that proxies serve containerboot's health check
// endpoint on. It is excluded from the egress config, which only describes user-facing ports.
tsHealthCheckPortName = "tailscale-health-check"
// maxPorts is the maximum number of ports that can be exposed on a
// container. In practice this will be ports in range [10000 - 11000). The
// high range should make it easier to distinguish container ports from
// the tailnet target ports for debugging purposes (i.e when reading
// netfilter rules). The limit of 1000 is somewhat arbitrary, the
// assumption is that this would not be hit in practice.
maxPorts = 1000
)
// CMName returns the name of the ConfigMap holding the egress service configs for the named ProxyGroup. The
// ProxyGroup reconciler creates it; the egress reconcilers read and update it.
func CMName(pg string) string {
return fmt.Sprintf("%s-egress-config", pg)
}
// childResourceLabels returns the labels applied to the ClusterIP Service and EndpointSlices created for the egress
// service backing svc.
//
// TODO(irbekrm): we currently set a bunch of labels based on Kubernetes
// resource names (ProxyGroup, Service). Maximum allowed label length is 63
// chars whilst the maximum allowed resource name length is 253 chars, so we
// should probably validate and truncate (?) the names is they are too long.
func childResourceLabels(svc *corev1.Service) map[string]string {
return map[string]string{
kubetypes.LabelManaged: "true",
reconciler.LabelParentType: parentTypeSvc,
reconciler.LabelParentName: svc.Name,
reconciler.LabelParentNamespace: svc.Namespace,
labelProxyGroup: svc.Annotations[reconciler.AnnotationProxyGroup],
labelSvcType: typeEgress,
}
}
// epsLabels returns the labels for an EndpointSlice created for an egress service, which are the child resource
// labels plus the two labels that make kube-proxy route ClusterIP Service traffic to this slice's endpoints.
func epsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string {
lbls := childResourceLabels(extNSvc)
// Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the
// endpoints defined on this EndpointSlice.
// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
lbls[discoveryv1.LabelServiceName] = clusterIPSvc.Name
// Kubernetes recommends setting this label.
// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management
lbls[discoveryv1.LabelManagedBy] = "tailscale.com"
return lbls
}
// tailnetSvcName returns the name used to distinguish the tailnet service exposed via extNSvc from the other tailnet
// services exposed to cluster workloads. It keys the egress config.
func tailnetSvcName(extNSvc *corev1.Service) string {
return fmt.Sprintf("%s-%s", extNSvc.Namespace, extNSvc.Name)
}
k8s-operator/reconciler/egress/egress_test.go
+61
View File
@@ -0,0 +1,61 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package egress
import (
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"tailscale.com/k8s-operator/reconciler"
)
// The names and labels below are a contract with things outside this package: proxies select on the labels, the
// ProxyGroup reconciler creates the ConfigMap CMName refers to, and the egress config is keyed by tailnetSvcName. The
// rest of the package's tests build their fixtures from these helpers, so they would follow a change here rather than
// catch it; assert the literal values once.
func TestCMName(t *testing.T) {
if got, want := CMName("pg"), "pg-egress-config"; got != want {
t.Errorf("CMName(\"pg\") = %q, want %q", got, want)
}
}
func TestTailnetSvcName(t *testing.T) {
svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "dev", Name: "my-app"}}
if got, want := tailnetSvcName(svc), "dev-my-app"; got != want {
t.Errorf("tailnetSvcName = %q, want %q", got, want)
}
}
func TestChildResourceLabels(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Namespace: "dev",
Name: "my-app",
Annotations: map[string]string{reconciler.AnnotationProxyGroup: "pg"},
},
}
want := map[string]string{
"tailscale.com/managed": "true",
"tailscale.com/parent-resource-type": "svc",
"tailscale.com/parent-resource": "my-app",
"tailscale.com/parent-resource-ns": "dev",
"tailscale.com/proxy-group": "pg",
"tailscale.com/svc-type": "egress",
}
got := childResourceLabels(svc)
if len(got) != len(want) {
t.Fatalf("got %d labels, want %d: %v", len(got), len(want), got)
}
for k, v := range want {
if got[k] != v {
t.Errorf("label %q = %q, want %q", k, got[k], v)
}
}
}
cmd/k8s-operator/egress-eps.go → k8s-operator/reconciler/egress/endpointslices.go
+38 -8
View File
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package egress
import (
"context"
@@ -18,24 +18,54 @@
discoveryv1 "k8s.io/api/discovery/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/egressservices"
)
// egressEpsReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies.
type egressEpsReconciler struct {
const endpointSliceReconcilerName = "egress-eps-reconciler"
// EndpointSliceReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies.
type EndpointSliceReconciler struct {
client.Client
logger *zap.SugaredLogger
tsNamespace string
}
// NewEndpointSliceReconciler returns the reconciler that keeps egress EndpointSlices pointing at the proxy Pods that
// can currently route to the tailnet target.
func NewEndpointSliceReconciler(opts Options) *EndpointSliceReconciler {
return &EndpointSliceReconciler{
Client: opts.Client,
logger: opts.Logger.Named(endpointSliceReconcilerName),
tsNamespace: opts.TailscaleNamespace,
}
}
// Register the EndpointSliceReconciler onto mgr. It watches the EndpointSlices it owns, the proxy Pods and their state
// Secrets (both of which determine whether a Pod can route), and the user's ExternalName Services.
func (er *EndpointSliceReconciler) Register(mgr manager.Manager) error {
return builder.
ControllerManagedBy(mgr).
Named(endpointSliceReconcilerName).
Watches(&discoveryv1.EndpointSlice{}, handler.EnqueueRequestsFromMapFunc(endpointSliceHandler)).
Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(endpointSlicesFromPods(er.Client, er.tsNamespace))).
Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(endpointSlicesFromStateSecrets(er.Client, er.tsNamespace))).
Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(endpointSlicesFromExternalNameService(er.Client, er.logger, er.tsNamespace))).
Complete(er)
}
// Reconcile reconciles an EndpointSlice for a tailnet service. It updates the EndpointSlice with the endpoints of
// those ProxyGroup Pods that are ready to route traffic to the tailnet service.
// It compares tailnet service state stored in egress proxy state Secrets by containerboot with the desired
// configuration stored in proxy-cfg ConfigMap to determine if the endpoint is ready.
func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
func (er *EndpointSliceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
lg := er.logger.With("Service", req.NamespacedName)
lg.Debugf("starting reconcile")
defer lg.Debugf("reconcile finished")
@@ -58,8 +88,8 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
// resources are set up for this tailnet service.
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: eps.Labels[LabelParentName],
Namespace: eps.Labels[LabelParentNamespace],
Name: eps.Labels[reconciler.LabelParentName],
Namespace: eps.Labels[reconciler.LabelParentNamespace],
},
}
err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc)
@@ -101,7 +131,7 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
// Check which Pods in ProxyGroup are ready to route traffic to this
// egress service.
podList := &corev1.PodList{}
if err := er.List(ctx, podList, client.MatchingLabels(pgLabels(proxyGroupName, nil))); err != nil {
if err := er.List(ctx, podList, client.MatchingLabels(reconciler.Labels("proxygroup", proxyGroupName, ""))); err != nil {
return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err)
}
newEndpoints := make([]discoveryv1.Endpoint, 0)
@@ -162,7 +192,7 @@ func podIPForFamily(pod *corev1.Pod, addrType discoveryv1.AddressType) (string,
// podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to
// route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service
// status written there to the desired service configuration.
func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, addrType discoveryv1.AddressType, lg *zap.SugaredLogger) (bool, error) {
func (er *EndpointSliceReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, addrType discoveryv1.AddressType, lg *zap.SugaredLogger) (bool, error) {
lg = lg.With("proxy_pod", pod.Name)
lg.Debug("checking whether proxy is ready to route to egress service")
if !pod.DeletionTimestamp.IsZero() {
cmd/k8s-operator/egress-eps_test.go → k8s-operator/reconciler/egress/endpointslices_test.go
+68 -57
View File
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package egress
import (
"encoding/json"
@@ -17,7 +17,10 @@
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
@@ -32,8 +35,8 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetFQDN: "foo.bar.ts.net",
AnnotationProxyGroup: "foo",
reconciler.AnnotationTailnetTargetFQDN: "foo.bar.ts.net",
reconciler.AnnotationProxyGroup: "foo",
},
},
Spec: corev1.ServiceSpec{
@@ -66,43 +69,43 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
if err != nil {
t.Fatal(err)
}
er := &egressEpsReconciler{
Client: fc,
logger: zl.Sugar(),
tsNamespace: "operator-ns",
}
er := NewEndpointSliceReconciler(Options{
Client: fc,
Logger: zl.Sugar(),
TailscaleNamespace: "operator-ns",
})
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: "operator-ns",
Labels: map[string]string{
LabelParentName: "test",
LabelParentNamespace: "default",
labelSvcType: typeEgress,
labelProxyGroup: "foo"},
reconciler.LabelParentName: "test",
reconciler.LabelParentNamespace: "default",
labelSvcType: typeEgress,
labelProxyGroup: "foo"},
},
AddressType: discoveryv1.AddressTypeIPv4,
}
mustCreate(t, fc, eps)
reconcilertest.MustCreate(t, fc, eps)
t.Run("no_proxy_group_resources", func(t *testing.T) {
expectReconciled(t, er, "operator-ns", "foo") // should not error
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo") // should not error
})
t.Run("no_pods_ready_to_route_traffic", func(t *testing.T) {
pod, stateS := podAndSecretForProxyGroup("foo")
mustCreate(t, fc, pod)
mustCreate(t, fc, stateS)
expectReconciled(t, er, "operator-ns", "foo") // should not error
reconcilertest.MustCreate(t, fc, pod)
reconcilertest.MustCreate(t, fc, stateS)
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo") // should not error
})
t.Run("pods_are_ready_to_route_traffic", func(t *testing.T) {
pod, stateS := podAndSecretForProxyGroup("foo")
stBs := serviceStatusForPodIPs(t, svc, pod.Status.PodIPs[0].IP, "", port)
mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
reconcilertest.MustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo")
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo")
eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
Addresses: []string{"10.0.0.1"},
Hostname: new("foo"),
@@ -112,17 +115,17 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
Terminating: new(false),
},
})
expectEqual(t, fc, eps)
reconcilertest.ExpectEqual(t, fc, eps)
})
t.Run("status_does_not_match_pod_ip", func(t *testing.T) {
_, stateS := podAndSecretForProxyGroup("foo") // replica Pod has IP 10.0.0.1
stBs := serviceStatusForPodIPs(t, svc, "10.0.0.2", "", port) // status is for a Pod with IP 10.0.0.2
mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
reconcilertest.MustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo")
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo")
eps.Endpoints = []discoveryv1.Endpoint{}
expectEqual(t, fc, eps)
reconcilertest.ExpectEqual(t, fc, eps)
})
// Dual-stack.
@@ -131,34 +134,34 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
Name: "foo-ipv6",
Namespace: "operator-ns",
Labels: map[string]string{
LabelParentName: "test",
LabelParentNamespace: "default",
labelSvcType: typeEgress,
labelProxyGroup: "foo",
reconciler.LabelParentName: "test",
reconciler.LabelParentNamespace: "default",
labelSvcType: typeEgress,
labelProxyGroup: "foo",
},
},
AddressType: discoveryv1.AddressTypeIPv6,
}
mustCreate(t, fc, epsV6)
reconcilertest.MustCreate(t, fc, epsV6)
t.Run("dual_stack_pod_ready_to_route", func(t *testing.T) {
mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
reconcilertest.MustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
dualPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "foo-0",
Namespace: "operator-ns",
Labels: pgLabels("foo", nil),
Labels: reconciler.Labels("proxygroup", "foo", ""),
UID: "foo",
},
Status: corev1.PodStatus{
PodIPs: []corev1.PodIP{{IP: "10.0.0.1"}, {IP: "fd00::1"}},
},
}
mustCreate(t, fc, dualPod)
reconcilertest.MustCreate(t, fc, dualPod)
stBs := serviceStatusForPodIPs(t, svc, "10.0.0.1", "fd00::1", port)
mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
reconcilertest.MustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo")
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo")
eps.Endpoints = []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.1"},
Hostname: new("foo"),
@@ -168,8 +171,8 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
Terminating: new(false),
},
}}
expectEqual(t, fc, eps)
expectReconciled(t, er, "operator-ns", "foo-ipv6")
reconcilertest.ExpectEqual(t, fc, eps)
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo-ipv6")
epsV6.Endpoints = []discoveryv1.Endpoint{{
Addresses: []string{"fd00::1"},
Hostname: new("foo"),
@@ -179,28 +182,28 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
Terminating: new(false),
},
}}
expectEqual(t, fc, epsV6)
reconcilertest.ExpectEqual(t, fc, epsV6)
})
// IPv6-only.
t.Run("ipv4_only_pod_skipped_for_ipv6_slice", func(t *testing.T) {
mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
reconcilertest.MustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
ipv4Pod, _ := podAndSecretForProxyGroup("foo")
mustCreate(t, fc, ipv4Pod)
reconcilertest.MustCreate(t, fc, ipv4Pod)
stBs := serviceStatusForPodIPs(t, svc, "10.0.0.1", "", port)
mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
reconcilertest.MustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo-ipv6")
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo-ipv6")
// IPv4-only pod should not appear in the IPv6 EndpointSlice.
epsV6.Endpoints = []discoveryv1.Endpoint{}
expectEqual(t, fc, epsV6)
reconcilertest.ExpectEqual(t, fc, epsV6)
})
ipv6Pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "foo-0",
Namespace: "operator-ns",
Labels: pgLabels("foo", nil),
Labels: reconciler.Labels("proxygroup", "foo", ""),
UID: "foo",
},
Status: corev1.PodStatus{
@@ -208,22 +211,22 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
},
}
t.Run("ipv6_status_does_not_match_pod_ip", func(t *testing.T) {
mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
mustCreate(t, fc, ipv6Pod)
reconcilertest.MustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
reconcilertest.MustCreate(t, fc, ipv6Pod)
stBs := serviceStatusForPodIPs(t, svc, "", "fd00::99", port)
mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
reconcilertest.MustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo-ipv6")
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo-ipv6")
epsV6.Endpoints = []discoveryv1.Endpoint{}
expectEqual(t, fc, epsV6)
reconcilertest.ExpectEqual(t, fc, epsV6)
})
t.Run("ipv6_pod_ready_to_route", func(t *testing.T) {
stBs := serviceStatusForPodIPs(t, svc, "", ipv6Pod.Status.PodIPs[0].IP, port)
mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
reconcilertest.MustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo-ipv6")
reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo-ipv6")
epsV6.Endpoints = append(epsV6.Endpoints, discoveryv1.Endpoint{
Addresses: []string{"fd00::1"},
Hostname: new("foo"),
@@ -233,7 +236,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
Terminating: new(false),
},
})
expectEqual(t, fc, epsV6)
reconcilertest.ExpectEqual(t, fc, epsV6)
})
}
@@ -246,10 +249,10 @@ func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.Config
cfg := egressservices.Config{
Ports: ports,
}
if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
if fqdn := svc.Annotations[reconciler.AnnotationTailnetTargetFQDN]; fqdn != "" {
cfg.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
}
if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
if ip := svc.Annotations[reconciler.AnnotationTailnetTargetIP]; ip != "" {
cfg.TailnetTarget = egressservices.TailnetTarget{IP: ip}
}
name := tailnetSvcName(svc)
@@ -260,7 +263,7 @@ func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.Config
}
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: pgEgressCMName(svc.Annotations[AnnotationProxyGroup]),
Name: CMName(svc.Annotations[reconciler.AnnotationProxyGroup]),
Namespace: "operator-ns",
},
BinaryData: map[string][]byte{egressservices.KeyEgressServices: bs},
@@ -275,10 +278,10 @@ func serviceStatusForPodIPs(t *testing.T, svc *corev1.Service, ipv4, ipv6 string
ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{}
}
svcSt := egressservices.ServiceStatus{Ports: ports}
if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
if fqdn := svc.Annotations[reconciler.AnnotationTailnetTargetFQDN]; fqdn != "" {
svcSt.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn}
}
if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" {
if ip := svc.Annotations[reconciler.AnnotationTailnetTargetIP]; ip != "" {
svcSt.TailnetTarget = egressservices.TailnetTarget{IP: ip}
}
svcName := tailnetSvcName(svc)
@@ -299,7 +302,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-0", pg),
Namespace: "operator-ns",
Labels: pgLabels(pg, nil),
Labels: reconciler.Labels("proxygroup", pg, ""),
UID: "foo",
},
Status: corev1.PodStatus{
@@ -312,7 +315,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-0", pg),
Namespace: "operator-ns",
Labels: pgSecretLabels(pg, kubetypes.LabelSecretTypeState),
Labels: stateSecretLabels(pg),
},
}
return p, s
@@ -321,3 +324,11 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
func randomPort() uint16 {
return uint16(rand.Int32N(1000) + 1000)
}
// stateSecretLabels returns the labels the ProxyGroup reconciler puts on a proxy's tailscaled state Secret. The egress
// reconcilers select on them to find which Pods are routing, so the fixtures have to match.
func stateSecretLabels(pgName string) map[string]string {
labels := reconciler.Labels("proxygroup", pgName, "")
labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeState
return labels
}
k8s-operator/reconciler/egress/index.go
+270
View File
@@ -0,0 +1,270 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package egress
import (
"context"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
)
// IndexProxyGroup is the field index name under which egress Services are indexed by the ProxyGroup they are exposed
// on, so that a ProxyGroup event can enqueue every Service it serves. Reconciler.Register installs it.
const IndexProxyGroup = ".metadata.annotations.egress-proxy-group"
// serviceHandler returns accepts a Kubernetes object and returns a reconcile
// request for it , if the object is a Tailscale egress Service meant to be
// exposed on a ProxyGroup.
func serviceHandler(_ context.Context, o client.Object) []reconcile.Request {
if !isEgressSvcForProxyGroup(o) {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: o.GetNamespace(),
Name: o.GetName(),
},
},
}
}
// servicesFromProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
// user-created ExternalName Services that should be exposed on this ProxyGroup.
func servicesFromProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
pg, ok := o.(*tsapi.ProxyGroup)
if !ok {
logger.Warn("ProxyGroup handler triggered for an object that is not a ProxyGroup")
return nil
}
if pg.Spec.Type != tsapi.ProxyGroupTypeEgress {
return nil
}
svcList := &corev1.ServiceList{}
if err := cl.List(ctx, svcList, client.MatchingFields{IndexProxyGroup: pg.Name}); err != nil {
logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, svc := range svcList.Items {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: svc.Namespace,
Name: svc.Name,
},
})
}
return reqs
}
}
// serviceFromEndpointSlice is an event handler for EndpointSlices. If an EndpointSlice is for an egress ExternalName Service
// meant to be exposed on a ProxyGroup, returns a reconcile request for the Service.
func serviceFromEndpointSlice(_ context.Context, o client.Object) []reconcile.Request {
if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
return nil
}
if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
svcName, ok := o.GetLabels()[reconciler.LabelParentName]
if !ok {
return nil
}
svcNs, ok := o.GetLabels()[reconciler.LabelParentNamespace]
if !ok {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: svcNs,
Name: svcName,
},
},
}
}
// endpointSliceHandler returns accepts an EndpointSlice and, if the EndpointSlice
// is for an egress service, returns a reconcile request for it.
func endpointSliceHandler(_ context.Context, o client.Object) []reconcile.Request {
if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: o.GetNamespace(),
Name: o.GetName(),
},
},
}
}
// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is,
// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
func endpointSlicesFromPods(cl client.Client, ns string) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
// have ingress ProxyGroups.
if typ := o.GetLabels()[reconciler.LabelParentType]; typ != "proxygroup" {
return nil
}
pg, ok := o.GetLabels()[reconciler.LabelParentName]
if !ok {
return nil
}
return endpointSliceRequestsForProxyGroup(ctx, pg, cl, ns)
}
}
// endpointSlicesFromStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is,
// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
func endpointSlicesFromStateSecrets(cl client.Client, ns string) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
if parentType := o.GetLabels()[reconciler.LabelParentType]; parentType != "proxygroup" {
return nil
}
if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState {
return nil
}
pg, ok := o.GetLabels()[reconciler.LabelParentName]
if !ok {
return nil
}
return endpointSliceRequestsForProxyGroup(ctx, pg, cl, ns)
}
}
// endpointSlicesFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
func endpointSlicesFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
svc, ok := o.(*corev1.Service)
if !ok {
logger.Warn("Service handler triggered for an object that is not a Service")
return nil
}
if !isEgressSvcForProxyGroup(svc) {
return nil
}
epsList := &discoveryv1.EndpointSliceList{}
if err := cl.List(ctx, epsList, client.InNamespace(ns),
client.MatchingLabels(childResourceLabels(svc))); err != nil {
logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, eps := range epsList.Items {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: eps.Namespace,
Name: eps.Name,
},
})
}
return reqs
}
}
func podsFromEndpointSlices(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
eps, ok := o.(*discoveryv1.EndpointSlice)
if !ok {
logger.Warn("EndpointSlice handler triggered for an object that is not a EndpointSlice")
return nil
}
if eps.Labels[labelProxyGroup] == "" {
return nil
}
if eps.Labels[labelSvcType] != "egress" {
return nil
}
podLabels := map[string]string{
kubetypes.LabelManaged: "true",
reconciler.LabelParentType: "proxygroup",
reconciler.LabelParentName: eps.Labels[labelProxyGroup],
}
podList := &corev1.PodList{}
if err := cl.List(ctx, podList, client.InNamespace(ns),
client.MatchingLabels(podLabels)); err != nil {
logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on EndpointSlice %s", err, eps.Name)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, pod := range podList.Items {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: pod.Namespace,
Name: pod.Name,
},
})
}
return reqs
}
}
func podHandler(_ context.Context, o client.Object) []reconcile.Request {
if typ := o.GetLabels()[reconciler.LabelParentType]; typ != proxyTypeProxyGroup {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: o.GetNamespace(),
Name: o.GetName(),
},
},
}
}
// IndexServices adds a local index to cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
// index is used a list filter.
func IndexServices(o client.Object) []string {
if !isEgressSvcForProxyGroup(o) {
return nil
}
return []string{o.GetAnnotations()[reconciler.AnnotationProxyGroup]}
}
func endpointSliceRequestsForProxyGroup(ctx context.Context, pg string, cl client.Client, ns string) []reconcile.Request {
epsList := discoveryv1.EndpointSliceList{}
if err := cl.List(ctx, &epsList,
client.InNamespace(ns),
client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
return nil
}
reqs := make([]reconcile.Request, 0)
for _, ep := range epsList.Items {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: ep.Namespace,
Name: ep.Name,
},
})
}
return reqs
}
cmd/k8s-operator/egress-pod-readiness.go → k8s-operator/reconciler/egress/pods.go
+55 -19
View File
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package egress
import (
"context"
@@ -17,32 +17,42 @@
"time"
"go.uber.org/zap"
xslices "golang.org/x/exp/slices"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstime"
"tailscale.com/util/backoff"
"tailscale.com/util/httpm"
)
const tsEgressReadinessGate = "tailscale.com/egress-services"
// ReadinessGate is the Pod readiness gate that PodReconciler sets once a proxy Pod has set up routing for its
// egress services. The ProxyGroup reconciler adds it to egress proxy Pod specs so that a rolling restart doesn't
// mark a Pod ready, and thus eligible for traffic, before it can actually route.
const ReadinessGate = "tailscale.com/egress-services"
// egressPodsReconciler is responsible for setting tailscale.com/egress-services condition on egress ProxyGroup Pods.
const podReconcilerName = "egress-pods-readiness-reconciler"
// PodReconciler is responsible for setting tailscale.com/egress-services condition on egress ProxyGroup Pods.
// The condition is used as a readiness gate for the Pod, meaning that kubelet will not mark the Pod as ready before the
// condition is set. The ProxyGroup StatefulSet updates are rolled out in such a way that no Pod is restarted, before
// the previous Pod is marked as ready, so ensuring that the Pod does not get marked as ready when it is not yet able to
// route traffic for egress service prevents downtime during restarts caused by no available endpoints left because
// every Pod has been recreated and is not yet added to endpoints.
// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate
type egressPodsReconciler struct {
type PodReconciler struct {
client.Client
logger *zap.SugaredLogger
tsNamespace string
clock tstime.Clock
@@ -50,6 +60,33 @@ type egressPodsReconciler struct {
maxBackoff time.Duration // max backoff period between health check calls
}
// NewPodReconciler returns the reconciler that gates egress proxy Pod readiness on the Pod having set up routing.
func NewPodReconciler(opts Options) *PodReconciler {
httpClient := opts.HTTPClient
if httpClient == nil {
httpClient = http.DefaultClient
}
return &PodReconciler{
Client: opts.Client,
logger: opts.Logger.Named(podReconcilerName),
tsNamespace: opts.TailscaleNamespace,
clock: opts.clock(),
httpClient: httpClient,
maxBackoff: opts.MaxBackoff,
}
}
// Register the PodReconciler onto mgr. It watches proxy Pods and the egress EndpointSlices that tell it which egress
// services a Pod is expected to be routing.
func (er *PodReconciler) Register(mgr manager.Manager) error {
return builder.
ControllerManagedBy(mgr).
Named(podReconcilerName).
Watches(&discoveryv1.EndpointSlice{}, handler.EnqueueRequestsFromMapFunc(podsFromEndpointSlices(er.Client, er.logger, er.tsNamespace))).
Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(podHandler)).
Complete(er)
}
// Reconcile reconciles an egress ProxyGroup Pods on changes to those Pods and ProxyGroup EndpointSlices. It ensures
// that for each Pod who is ready to route traffic to all egress services for the ProxyGroup, the Pod has a
// tailscale.com/egress-services condition to set, so that kubelet will mark the Pod as ready.
@@ -72,7 +109,7 @@ type egressPodsReconciler struct {
//
// If the Pod does not appear to be serving the health check endpoint (pre-v1.80 proxies), the reconciler just sets the
// readiness condition for backwards compatibility reasons.
func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
func (er *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
lg := er.logger.With("Pod", req.NamespacedName)
lg.Debugf("starting reconcile")
defer lg.Debugf("reconcile finished")
@@ -90,7 +127,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
return res, nil
}
if pod.Labels[LabelParentType] != proxyTypeProxyGroup {
if pod.Labels[reconciler.LabelParentType] != proxyTypeProxyGroup {
lg.Warn("reconciler called for a Pod that is not a ProxyGroup Pod")
return res, nil
}
@@ -98,13 +135,13 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
// If the Pod does not have the readiness gate set, there is no need to add the readiness condition. In practice
// this will happen if the user has configured custom TS_LOCAL_ADDR_PORT, thus disabling the graceful failover.
if !slices.ContainsFunc(pod.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool {
return r.ConditionType == tsEgressReadinessGate
return r.ConditionType == ReadinessGate
}) {
lg.Debug("Pod does not have egress readiness gate set, skipping")
return res, nil
}
proxyGroupName := pod.Labels[LabelParentName]
proxyGroupName := pod.Labels[reconciler.LabelParentName]
pg := new(tsapi.ProxyGroup)
if err := er.Get(ctx, types.NamespacedName{Name: proxyGroupName}, pg); err != nil {
return res, fmt.Errorf("error getting ProxyGroup %q: %w", proxyGroupName, err)
@@ -126,10 +163,9 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
return res, fmt.Errorf("error listing ClusterIP Services")
}
idx := xslices.IndexFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool {
return c.Type == tsEgressReadinessGate
})
if idx != -1 {
if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool {
return c.Type == ReadinessGate
}) {
lg.Debugf("Pod is already ready, do nothing")
return res, nil
}
@@ -140,7 +176,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
s := svc
go func() {
ll := lg.With("service_name", s.Name)
d := retrieveClusterDomain(er.tsNamespace, ll)
d := reconciler.ClusterDomain(er.tsNamespace, ll)
healthCheckAddr := healthCheckForSvc(&s, d)
if healthCheckAddr == "" {
ll.Debugf("ClusterIP Service does not expose a health check endpoint, unable to verify if routing is set up")
@@ -150,7 +186,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
var routesSetup bool
bo := backoff.NewBackoff(s.Name, ll.Infof, er.maxBackoff)
for range numCalls(pgReplicas(pg)) {
for range numCalls(reconciler.ProxyGroupReplicas(pg)) {
if ctx.Err() != nil {
errChan <- nil
return
@@ -192,15 +228,15 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
return res, nil
}
func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, lg *zap.SugaredLogger) error {
func (er *PodReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, lg *zap.SugaredLogger) error {
if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool {
return c.Type == tsEgressReadinessGate
return c.Type == ReadinessGate
}) {
return nil
}
lg.Infof("Pod is ready to route traffic to all egress targets")
pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
Type: tsEgressReadinessGate,
Type: ReadinessGate,
Status: corev1.ConditionTrue,
LastTransitionTime: metav1.Time{Time: er.clock.Now()},
})
@@ -221,7 +257,7 @@ func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod
)
// lookupPodRouteViaSvc attempts to reach a Pod using a health check endpoint served by a Service and returns the state of the health check.
func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, lg *zap.SugaredLogger) (healthCheckState, error) {
func (er *PodReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, lg *zap.SugaredLogger) (healthCheckState, error) {
if !slices.ContainsFunc(pod.Spec.Containers[0].Env, func(e corev1.EnvVar) bool {
return e.Name == "TS_ENABLE_HEALTH_CHECK" && e.Value == "true"
}) {
cmd/k8s-operator/egress-pod-readiness_test.go → k8s-operator/reconciler/egress/pods_test.go
+98 -86
View File
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package egress
import (
"bytes"
@@ -21,7 +21,10 @@
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
)
@@ -35,12 +38,12 @@ func TestEgressPodReadiness(t *testing.T) {
Build()
zl, _ := zap.NewDevelopment()
cl := tstest.NewClock(tstest.ClockOpts{})
rec := &egressPodsReconciler{
tsNamespace: "operator-ns",
Client: fc,
logger: zl.Sugar(),
clock: cl,
}
rec := NewPodReconciler(Options{
TailscaleNamespace: "operator-ns",
Client: fc,
Logger: zl.Sugar(),
Clock: cl,
})
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "dev",
@@ -50,20 +53,20 @@ func TestEgressPodReadiness(t *testing.T) {
Replicas: new(int32(3)),
},
}
mustCreate(t, fc, pg)
reconcilertest.MustCreate(t, fc, pg)
podIP := "10.0.0.2"
podTemplate := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "operator-ns",
Name: "pod",
Labels: map[string]string{
LabelParentType: "proxygroup",
LabelParentName: "dev",
reconciler.LabelParentType: "proxygroup",
reconciler.LabelParentName: "dev",
},
},
Spec: corev1.PodSpec{
ReadinessGates: []corev1.PodReadinessGate{{
ConditionType: tsEgressReadinessGate,
ConditionType: ReadinessGate,
}},
Containers: []corev1.Container{{
Name: "tailscale",
@@ -80,42 +83,42 @@ func TestEgressPodReadiness(t *testing.T) {
t.Run("no_egress_services", func(t *testing.T) {
pod := podTemplate.DeepCopy()
mustCreate(t, fc, pod)
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.MustCreate(t, fc, pod)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod)
})
t.Run("one_svc_already_routed_to", func(t *testing.T) {
pod := podTemplate.DeepCopy()
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
resp := readyResps(podIP, 1)
httpCl := fakeHTTPClient{
t: t,
state: map[string][]fakeResponse{hep: resp},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
reconcilertest.ExpectEqual(t, fc, pod)
// A subsequent reconcile should not change the Pod.
expectReconciled(t, rec, "operator-ns", pod.Name)
expectEqual(t, fc, pod)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
t.Run("one_svc_many_backends_eventually_routed_to", func(t *testing.T) {
pod := podTemplate.DeepCopy()
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only
// return with the right Pod IP.
resps := append(readyResps("10.0.0.3", 4), append(readyResps("10.0.0.4", 4), readyResps(podIP, 1)...)...)
@@ -124,18 +127,18 @@ func TestEgressPodReadiness(t *testing.T) {
state: map[string][]fakeResponse{hep: resps},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
t.Run("one_svc_one_backend_eventually_healthy", func(t *testing.T) {
pod := podTemplate.DeepCopy()
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only
// return with 200 status code.
resps := append(unreadyResps(podIP, 8), readyResps(podIP, 1)...)
@@ -144,18 +147,18 @@ func TestEgressPodReadiness(t *testing.T) {
state: map[string][]fakeResponse{hep: resps},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
t.Run("one_svc_one_backend_never_routable", func(t *testing.T) {
pod := podTemplate.DeepCopy()
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
// For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times and Pod should be
// requeued if neither of those succeed.
resps := readyResps("10.0.0.3", 9)
@@ -164,11 +167,11 @@ func TestEgressPodReadiness(t *testing.T) {
state: map[string][]fakeResponse{hep: resps},
}
rec.httpClient = &httpCl
expectRequeue(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name)
// Pod should not have readiness gate condition set.
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
t.Run("one_svc_many_backends_already_routable", func(t *testing.T) {
pod := podTemplate.DeepCopy()
@@ -176,7 +179,7 @@ func TestEgressPodReadiness(t *testing.T) {
svc, hep := newSvc("svc", 9002)
svc2, hep2 := newSvc("svc-2", 9002)
svc3, hep3 := newSvc("svc-3", 9002)
mustCreateAll(t, fc, svc, svc2, svc3, pod)
reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod)
resps := readyResps(podIP, 1)
httpCl := fakeHTTPClient{
t: t,
@@ -187,19 +190,19 @@ func TestEgressPodReadiness(t *testing.T) {
},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should not have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
t.Run("one_svc_many_backends_eventually_routable_and_healthy", func(t *testing.T) {
pod := podTemplate.DeepCopy()
svc, hep := newSvc("svc", 9002)
svc2, hep2 := newSvc("svc-2", 9002)
svc3, hep3 := newSvc("svc-3", 9002)
mustCreateAll(t, fc, svc, svc2, svc3, pod)
reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod)
resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...)
resps2 := append(readyResps("10.0.0.3", 5), readyResps(podIP, 1)...)
resps3 := append(unreadyResps(podIP, 4), readyResps(podIP, 1)...)
@@ -212,12 +215,12 @@ func TestEgressPodReadiness(t *testing.T) {
},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
t.Run("one_svc_many_backends_never_routable_and_healthy", func(t *testing.T) {
pod := podTemplate.DeepCopy()
@@ -225,7 +228,7 @@ func TestEgressPodReadiness(t *testing.T) {
svc, hep := newSvc("svc", 9002)
svc2, hep2 := newSvc("svc-2", 9002)
svc3, hep3 := newSvc("svc-3", 9002)
mustCreateAll(t, fc, svc, svc2, svc3, pod)
reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod)
// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
// will be requeued if neither succeeds.
resps := readyResps("10.0.0.3", 9)
@@ -240,11 +243,11 @@ func TestEgressPodReadiness(t *testing.T) {
},
}
rec.httpClient = &httpCl
expectRequeue(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name)
// Pod should not have readiness gate condition set.
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
t.Run("one_svc_many_backends_one_never_routable", func(t *testing.T) {
pod := podTemplate.DeepCopy()
@@ -252,7 +255,7 @@ func TestEgressPodReadiness(t *testing.T) {
svc, hep := newSvc("svc", 9002)
svc2, hep2 := newSvc("svc-2", 9002)
svc3, hep3 := newSvc("svc-3", 9002)
mustCreateAll(t, fc, svc, svc2, svc3, pod)
reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod)
// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
// will be requeued if any one never succeeds.
resps := readyResps(podIP, 9)
@@ -267,11 +270,11 @@ func TestEgressPodReadiness(t *testing.T) {
},
}
rec.httpClient = &httpCl
expectRequeue(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name)
// Pod should not have readiness gate condition set.
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
t.Run("one_svc_many_backends_one_never_healthy", func(t *testing.T) {
pod := podTemplate.DeepCopy()
@@ -279,7 +282,7 @@ func TestEgressPodReadiness(t *testing.T) {
svc, hep := newSvc("svc", 9002)
svc2, hep2 := newSvc("svc-2", 9002)
svc3, hep3 := newSvc("svc-3", 9002)
mustCreateAll(t, fc, svc, svc2, svc3, pod)
reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod)
// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod
// will be requeued if any one never succeeds.
resps := readyResps(podIP, 9)
@@ -294,11 +297,11 @@ func TestEgressPodReadiness(t *testing.T) {
},
}
rec.httpClient = &httpCl
expectRequeue(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name)
// Pod should not have readiness gate condition set.
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
t.Run("one_svc_many_backends_different_ports_eventually_healthy_and_routable", func(t *testing.T) {
pod := podTemplate.DeepCopy()
@@ -306,7 +309,7 @@ func TestEgressPodReadiness(t *testing.T) {
svc, hep := newSvc("svc", 9003)
svc2, hep2 := newSvc("svc-2", 9004)
svc3, hep3 := newSvc("svc-3", 9010)
mustCreateAll(t, fc, svc, svc2, svc3, pod)
reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod)
// For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried up to 9 times and
// marked as success as soon as one try succeeds.
resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...)
@@ -321,12 +324,12 @@ func TestEgressPodReadiness(t *testing.T) {
},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
// Proxies of 1.78 and earlier did not set the Pod IP header.
t.Run("pod_does_not_return_ip_header", func(t *testing.T) {
@@ -334,7 +337,7 @@ func TestEgressPodReadiness(t *testing.T) {
pod.Name = "foo-bar"
svc, hep := newSvc("foo-bar", 9002)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
// If a response does not contain Pod IP header, we assume that this is an earlier proxy version,
// readiness cannot be verified so the readiness gate is just set to true.
resps := unreadyResps("", 1)
@@ -345,18 +348,18 @@ func TestEgressPodReadiness(t *testing.T) {
},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
t.Run("one_svc_one_backend_eventually_healthy_and_routable", func(t *testing.T) {
pod := podTemplate.DeepCopy()
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
// If a response errors, it is probably because the Pod is not yet properly running, so retry.
resps := append(erroredResps(8), readyResps(podIP, 1)...)
httpCl := fakeHTTPClient{
@@ -366,12 +369,12 @@ func TestEgressPodReadiness(t *testing.T) {
},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
t.Run("one_svc_one_backend_svc_does_not_have_health_port", func(t *testing.T) {
pod := podTemplate.DeepCopy()
@@ -379,14 +382,14 @@ func TestEgressPodReadiness(t *testing.T) {
// If a Service does not have health port set, we assume that it is not possible to determine Pod's
// readiness and set it to ready.
svc, _ := newSvc("svc", -1)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
rec.httpClient = nil
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
// Pod should have readiness gate condition set.
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
t.Run("error_setting_up_healthcheck", func(t *testing.T) {
pod := podTemplate.DeepCopy()
@@ -397,13 +400,13 @@ func TestEgressPodReadiness(t *testing.T) {
svc, _ := newSvc("svc", 9002)
svc2, _ := newSvc("svc-2", 9002)
svc3, _ := newSvc("svc-3", 9002)
mustCreateAll(t, fc, svc, svc2, svc3, pod)
reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod)
rec.httpClient = nil
expectError(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconcileError(t, rec, "operator-ns", pod.Name)
// Pod should not have readiness gate condition set.
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
t.Run("pod_does_not_have_an_ip_address", func(t *testing.T) {
pod := podTemplate.DeepCopy()
@@ -412,38 +415,38 @@ func TestEgressPodReadiness(t *testing.T) {
svc, _ := newSvc("svc", 9002)
svc2, _ := newSvc("svc-2", 9002)
svc3, _ := newSvc("svc-3", 9002)
mustCreateAll(t, fc, svc, svc2, svc3, pod)
reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod)
rec.httpClient = nil
expectRequeue(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name)
// Pod should not have readiness gate condition set.
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
t.Run("ipv6_only_pod_already_routed_to", func(t *testing.T) {
pod := podTemplate.DeepCopy()
pod.Status.PodIPs = []corev1.PodIP{{IP: "fd00::2"}}
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
resp := readyRespsV6("fd00::2", 1)
httpCl := fakeHTTPClient{
t: t,
state: map[string][]fakeResponse{hep: resp},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
t.Run("dual_stack_pod", func(t *testing.T) {
pod := podTemplate.DeepCopy()
pod.Status.PodIPs = []corev1.PodIP{{IP: "10.0.0.2"}, {IP: "fd00::2"}}
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
reconcilertest.MustCreateAll(t, fc, svc, pod)
// Dual-stack pod: the reconciler uses PodIPs[0] (the primary IP),
// which in this case is IPv4.
resp := readyResps("10.0.0.2", 1)
@@ -452,11 +455,11 @@ func TestEgressPodReadiness(t *testing.T) {
state: map[string][]fakeResponse{hep: resp},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name)
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
reconcilertest.ExpectEqual(t, fc, pod)
reconcilertest.MustDeleteAll(t, fc, pod, svc)
})
}
@@ -516,7 +519,7 @@ func newSvc(name string, port int32) (*corev1.Service, string) {
func podSetReady(pod *corev1.Pod, cl *tstest.Clock) {
pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
Type: tsEgressReadinessGate,
Type: ReadinessGate,
Status: corev1.ConditionTrue,
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
})
@@ -572,3 +575,12 @@ type fakeResponse struct {
podIP string // for the Pod IP header
header string // header key to use; defaults to PodIPv4Header
}
// TestReadinessGateName pins the readiness gate's wire value. The ProxyGroup reconciler stamps it onto egress proxy
// Pod specs and PodReconciler is what satisfies it, so the two must agree; every other test in this package builds its
// fixtures from the constant and so would follow a rename rather than catch it.
func TestReadinessGateName(t *testing.T) {
if got, want := ReadinessGate, "tailscale.com/egress-services"; got != want {
t.Errorf("ReadinessGate = %q, want %q", got, want)
}
}
cmd/k8s-operator/egress-services-readiness.go → k8s-operator/reconciler/egress/readiness.go
+39 -12
View File
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package egress
import (
"context"
@@ -19,11 +19,14 @@
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/tstime"
"tailscale.com/util/set"
)
@@ -38,18 +41,42 @@
msgReadyToRouteTemplate = "%d out of %d replicas are ready to route traffic"
)
type egressSvcsReadinessReconciler struct {
const readinessReconcilerName = "egress-svcs-readiness-reconciler"
type ReadinessReconciler struct {
client.Client
logger *zap.SugaredLogger
clock tstime.Clock
tsNamespace string
}
// NewReadinessReconciler returns the reconciler that surfaces egress Service readiness.
func NewReadinessReconciler(opts Options) *ReadinessReconciler {
return &ReadinessReconciler{
Client: opts.Client,
logger: opts.Logger.Named(readinessReconcilerName),
clock: opts.clock(),
tsNamespace: opts.TailscaleNamespace,
}
}
// Register the ReadinessReconciler onto mgr. It watches egress Services and the EndpointSlices backing them, since
// readiness is derived from whether any endpoint is serving.
func (esrr *ReadinessReconciler) Register(mgr manager.Manager) error {
return builder.
ControllerManagedBy(mgr).
Named(readinessReconcilerName).
Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandler)).
Watches(&discoveryv1.EndpointSlice{}, handler.EnqueueRequestsFromMapFunc(serviceFromEndpointSlice)).
Complete(esrr)
}
// Reconcile reconciles an ExternalName Service that defines a tailnet target to be exposed on a ProxyGroup and sets the
// EgressSvcReady condition on it. The condition gets set to true if at least one of the proxies is currently ready to
// route traffic to the target. It compares proxy Pod IPs with the endpoints set on the EndpointSlice for the egress
// service to determine how many replicas are currently able to route traffic.
func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
func (esrr *ReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
lg := esrr.logger.With("Service", req.NamespacedName)
lg.Debugf("starting reconcile")
defer lg.Debugf("reconcile finished")
@@ -67,13 +94,13 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
)
oldStatus := svc.Status.DeepCopy()
defer func() {
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, lg)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, lg)
if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) {
err = errors.Join(err, esrr.Status().Update(ctx, svc))
}
}()
crl := egressSvcChildResourceLabels(svc)
crl := childResourceLabels(svc)
epsList := &discoveryv1.EndpointSliceList{}
if err = esrr.List(ctx, epsList, client.InNamespace(esrr.tsNamespace), client.MatchingLabels(crl)); err != nil {
err = fmt.Errorf("error listing EndpointSlices: %w", err)
@@ -94,7 +121,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
// recreated when this status change re-triggers a Service reconcile.
//
// TODO(beckypauley): refactor so EndpointSlice recovery is not dependent on Service status.
clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esrr.Client, esrr.tsNamespace, crl)
clusterIPSvc, err := reconciler.GetSingleObject[corev1.Service](ctx, esrr.Client, esrr.tsNamespace, crl)
if err != nil {
err = fmt.Errorf("error retrieving ClusterIP Service: %w", err)
reason = reasonReadinessCheckFailed
@@ -127,7 +154,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
}
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: svc.Annotations[AnnotationProxyGroup],
Name: svc.Annotations[reconciler.AnnotationProxyGroup],
},
}
err = esrr.Get(ctx, client.ObjectKeyFromObject(pg), pg)
@@ -143,26 +170,26 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
msg = err.Error()
return res, err
}
if !tsoperator.ProxyGroupAvailable(pg) {
if !reconciler.ProxyGroupAvailable(pg) {
lg.Infof("ProxyGroup for Service is not ready, waiting...")
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
st = metav1.ConditionFalse
return res, nil
}
replicas := pgReplicas(pg)
replicas := reconciler.ProxyGroupReplicas(pg)
if replicas == 0 {
lg.Infof("ProxyGroup replicas set to 0")
reason, msg = reasonNoProxies, reasonNoProxies
st = metav1.ConditionFalse
return res, nil
}
podLabels := pgLabels(pg.Name, nil)
podLabels := reconciler.Labels("proxygroup", pg.Name, "")
var readyReplicas int32
nextReplica:
for i := range replicas {
podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i)
pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels)
pod, err := reconciler.GetSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels)
if err != nil {
err = fmt.Errorf("error retrieving ProxyGroup Pod: %w", err)
reason = reasonReadinessCheckFailed
cmd/k8s-operator/egress-services-readiness_test.go → k8s-operator/reconciler/egress/readiness_test.go
+102 -100
View File
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package egress
import (
"fmt"
@@ -15,8 +15,10 @@
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/tstest"
"tailscale.com/tstime"
)
@@ -30,20 +32,20 @@ func TestEgressServiceReadiness(t *testing.T) {
Build()
zl, _ := zap.NewDevelopment()
cl := tstest.NewClock(tstest.ClockOpts{})
rec := &egressSvcsReadinessReconciler{
tsNamespace: "operator-ns",
Client: fc,
logger: zl.Sugar(),
clock: cl,
}
rec := NewReadinessReconciler(Options{
TailscaleNamespace: "operator-ns",
Client: fc,
Logger: zl.Sugar(),
Clock: cl,
})
tailnetFQDN := "my-app.tailnetxyz.ts.net"
egressSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "dev",
Annotations: map[string]string{
AnnotationProxyGroup: "dev",
AnnotationTailnetTargetFQDN: tailnetFQDN,
reconciler.AnnotationProxyGroup: "dev",
reconciler.AnnotationTailnetTargetFQDN: tailnetFQDN,
},
},
}
@@ -51,11 +53,11 @@ func TestEgressServiceReadiness(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "operator-ns",
Labels: egressSvcChildResourceLabels(egressSvc),
Labels: childResourceLabels(egressSvc),
},
Spec: corev1.ServiceSpec{ClusterIPs: []string{"10.0.0.1"}},
}
labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
labels := epsLabels(egressSvc, fakeClusterIPSvc)
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
@@ -69,59 +71,59 @@ func TestEgressServiceReadiness(t *testing.T) {
Name: "dev",
},
}
mustCreate(t, fc, egressSvc)
mustCreate(t, fc, fakeClusterIPSvc)
reconcilertest.MustCreate(t, fc, egressSvc)
reconcilertest.MustCreate(t, fc, fakeClusterIPSvc)
setClusterNotReady(egressSvc, cl, zl.Sugar())
t.Run("endpointslice_does_not_exist", func(t *testing.T) {
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // not ready
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
reconcilertest.ExpectEqual(t, fc, egressSvc) // not ready
})
t.Run("proxy_group_does_not_exist", func(t *testing.T) {
mustCreate(t, fc, eps)
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // still not ready
reconcilertest.MustCreate(t, fc, eps)
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
reconcilertest.ExpectEqual(t, fc, egressSvc) // still not ready
})
t.Run("proxy_group_not_ready", func(t *testing.T) {
mustCreate(t, fc, pg)
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // still not ready
reconcilertest.MustCreate(t, fc, pg)
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
reconcilertest.ExpectEqual(t, fc, egressSvc) // still not ready
})
t.Run("no_ready_replicas", func(t *testing.T) {
setPGReady(pg, cl, zl.Sugar())
mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
reconcilertest.MustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
p.Status = pg.Status
})
expectEqual(t, fc, pg)
for i := range pgReplicas(pg) {
reconcilertest.ExpectEqual(t, fc, pg)
for i := range reconciler.ProxyGroupReplicas(pg) {
p := pod(pg, i)
mustCreate(t, fc, p)
mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
reconcilertest.MustCreate(t, fc, p)
reconcilertest.MustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
existing.Status.PodIPs = p.Status.PodIPs
})
}
expectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
expectEqual(t, fc, egressSvc) // still not ready
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg))
reconcilertest.ExpectEqual(t, fc, egressSvc) // still not ready
})
t.Run("one_ready_replica", func(t *testing.T) {
setEndpointForReplica(pg, 0, eps)
mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
reconcilertest.MustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
e.Endpoints = eps.Endpoints
})
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1)
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // partially ready
setReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg), 1)
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
reconcilertest.ExpectEqual(t, fc, egressSvc) // partially ready
})
t.Run("all_replicas_ready", func(t *testing.T) {
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
setEndpointForReplica(pg, i, eps)
}
mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
reconcilertest.MustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
e.Endpoints = eps.Endpoints
})
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // ready
setReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg), reconciler.ProxyGroupReplicas(pg))
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
reconcilertest.ExpectEqual(t, fc, egressSvc) // ready
})
}
@@ -132,20 +134,20 @@ func TestEgressServiceReadinessDualStack(t *testing.T) {
Build()
zl, _ := zap.NewDevelopment()
cl := tstest.NewClock(tstest.ClockOpts{})
rec := &egressSvcsReadinessReconciler{
tsNamespace: "operator-ns",
Client: fc,
logger: zl.Sugar(),
clock: cl,
}
rec := NewReadinessReconciler(Options{
TailscaleNamespace: "operator-ns",
Client: fc,
Logger: zl.Sugar(),
Clock: cl,
})
tailnetFQDN := "my-app.tailnetxyz.ts.net"
egressSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "dev",
Annotations: map[string]string{
AnnotationProxyGroup: "dev",
AnnotationTailnetTargetFQDN: tailnetFQDN,
reconciler.AnnotationProxyGroup: "dev",
reconciler.AnnotationTailnetTargetFQDN: tailnetFQDN,
},
},
}
@@ -153,11 +155,11 @@ func TestEgressServiceReadinessDualStack(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "operator-ns",
Labels: egressSvcChildResourceLabels(egressSvc),
Labels: childResourceLabels(egressSvc),
},
Spec: corev1.ServiceSpec{ClusterIPs: []string{"10.0.0.1", "fd00::1"}},
}
labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
labels := epsLabels(egressSvc, fakeClusterIPSvc)
epsV4 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app-ipv4",
@@ -166,7 +168,7 @@ func TestEgressServiceReadinessDualStack(t *testing.T) {
},
AddressType: discoveryv1.AddressTypeIPv4,
}
labelsV6 := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
labelsV6 := epsLabels(egressSvc, fakeClusterIPSvc)
epsV6 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app-ipv6",
@@ -184,41 +186,41 @@ func TestEgressServiceReadinessDualStack(t *testing.T) {
Type: tsapi.ProxyGroupTypeEgress,
},
}
mustCreate(t, fc, egressSvc)
mustCreate(t, fc, fakeClusterIPSvc)
mustCreate(t, fc, epsV4)
mustCreate(t, fc, epsV6)
mustCreate(t, fc, pg)
reconcilertest.MustCreate(t, fc, egressSvc)
reconcilertest.MustCreate(t, fc, fakeClusterIPSvc)
reconcilertest.MustCreate(t, fc, epsV4)
reconcilertest.MustCreate(t, fc, epsV6)
reconcilertest.MustCreate(t, fc, pg)
setPGReady(pg, cl, zl.Sugar())
mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
reconcilertest.MustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
p.Status = pg.Status
})
// Create a dual-stack pod.
p := pod(pg, 0)
p.Status.PodIPs = append(p.Status.PodIPs, corev1.PodIP{IP: "fd00::0"})
mustCreate(t, fc, p)
mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
reconcilertest.MustCreate(t, fc, p)
reconcilertest.MustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
existing.Status.PodIPs = p.Status.PodIPs
})
t.Run("not_ready_missing_from_ipv6_slice", func(t *testing.T) {
setEndpointForReplicaWithIP("10.0.0.0", epsV4)
mustUpdate(t, fc, epsV4.Namespace, epsV4.Name, func(e *discoveryv1.EndpointSlice) {
reconcilertest.MustUpdate(t, fc, epsV4.Namespace, epsV4.Name, func(e *discoveryv1.EndpointSlice) {
e.Endpoints = epsV4.Endpoints
})
expectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
expectEqual(t, fc, egressSvc)
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg))
reconcilertest.ExpectEqual(t, fc, egressSvc)
})
t.Run("ready_in_both_slices", func(t *testing.T) {
setEndpointForReplicaWithIP("fd00::", epsV6)
mustUpdate(t, fc, epsV6.Namespace, epsV6.Name, func(e *discoveryv1.EndpointSlice) {
reconcilertest.MustUpdate(t, fc, epsV6.Namespace, epsV6.Name, func(e *discoveryv1.EndpointSlice) {
e.Endpoints = epsV6.Endpoints
})
expectReconciled(t, rec, "dev", "my-app")
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
expectEqual(t, fc, egressSvc)
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
setReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg), reconciler.ProxyGroupReplicas(pg))
reconcilertest.ExpectEqual(t, fc, egressSvc)
})
t.Run("not_ready_when_ipv6_slice_missing", func(t *testing.T) {
// Delete the IPv6 EndpointSlice while the ClusterIP Service still
@@ -227,9 +229,9 @@ func TestEgressServiceReadinessDualStack(t *testing.T) {
if err := fc.Delete(t.Context(), epsV6); err != nil {
t.Fatalf("error deleting IPv6 EndpointSlice: %v", err)
}
expectReconciled(t, rec, "dev", "my-app")
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
setClusterNotReady(egressSvc, cl, zl.Sugar())
expectEqual(t, fc, egressSvc)
reconcilertest.ExpectEqual(t, fc, egressSvc)
})
}
@@ -240,19 +242,19 @@ func TestEgressServiceReadinessIPv6Only(t *testing.T) {
Build()
zl, _ := zap.NewDevelopment()
cl := tstest.NewClock(tstest.ClockOpts{})
rec := &egressSvcsReadinessReconciler{
tsNamespace: "operator-ns",
Client: fc,
logger: zl.Sugar(),
clock: cl,
}
rec := NewReadinessReconciler(Options{
TailscaleNamespace: "operator-ns",
Client: fc,
Logger: zl.Sugar(),
Clock: cl,
})
egressSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "dev",
Annotations: map[string]string{
AnnotationProxyGroup: "dev",
AnnotationTailnetTargetFQDN: "my-app.tailnetxyz.ts.net",
reconciler.AnnotationProxyGroup: "dev",
reconciler.AnnotationTailnetTargetFQDN: "my-app.tailnetxyz.ts.net",
},
},
}
@@ -260,11 +262,11 @@ func TestEgressServiceReadinessIPv6Only(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "operator-ns",
Labels: egressSvcChildResourceLabels(egressSvc),
Labels: childResourceLabels(egressSvc),
},
Spec: corev1.ServiceSpec{ClusterIPs: []string{"fd00::1"}},
}
labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
labels := epsLabels(egressSvc, fakeClusterIPSvc)
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app-ipv6",
@@ -278,45 +280,45 @@ func TestEgressServiceReadinessIPv6Only(t *testing.T) {
Name: "dev",
},
}
mustCreate(t, fc, egressSvc)
mustCreate(t, fc, fakeClusterIPSvc)
mustCreate(t, fc, eps)
mustCreate(t, fc, pg)
reconcilertest.MustCreate(t, fc, egressSvc)
reconcilertest.MustCreate(t, fc, fakeClusterIPSvc)
reconcilertest.MustCreate(t, fc, eps)
reconcilertest.MustCreate(t, fc, pg)
setPGReady(pg, cl, zl.Sugar())
mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
reconcilertest.MustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
p.Status = pg.Status
})
// Create IPv6-only pods.
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
p := ipv6OnlyPod(pg, i)
mustCreate(t, fc, p)
mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
reconcilertest.MustCreate(t, fc, p)
reconcilertest.MustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
existing.Status.PodIPs = p.Status.PodIPs
})
}
t.Run("no_ready_replicas", func(t *testing.T) {
expectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
expectEqual(t, fc, egressSvc)
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg))
reconcilertest.ExpectEqual(t, fc, egressSvc)
})
t.Run("all_replicas_ready", func(t *testing.T) {
for i := range pgReplicas(pg) {
for i := range reconciler.ProxyGroupReplicas(pg) {
p := ipv6OnlyPod(pg, i)
setEndpointForReplicaWithIP(p.Status.PodIPs[0].IP, eps)
}
mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
reconcilertest.MustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
e.Endpoints = eps.Endpoints
})
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc)
setReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg), reconciler.ProxyGroupReplicas(pg))
reconcilertest.ExpectReconciled(t, rec, "dev", "my-app")
reconcilertest.ExpectEqual(t, fc, egressSvc)
})
}
func ipv6OnlyPod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
labels := pgLabels(pg.Name, nil)
labels := reconciler.Labels("proxygroup", pg.Name, "")
labels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal)
ip := fmt.Sprintf("fd00::%d", ordinal+1) // +1 to avoid fd00::0 normalization issues
return &corev1.Pod{
@@ -332,12 +334,12 @@ func ipv6OnlyPod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
}
func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger) {
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, lg)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, lg)
}
func setNotReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger, replicas int32) {
msg := fmt.Sprintf(msgReadyToRouteTemplate, 0, replicas)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, lg)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, lg)
}
func setReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger, replicas, readyReplicas int32) {
@@ -346,11 +348,11 @@ func setReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger, repli
reason = reasonReady
}
msg := fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, lg)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, lg)
}
func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, lg *zap.SugaredLogger) {
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, lg)
reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, lg)
}
func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) {
@@ -366,7 +368,7 @@ func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1
}
func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
labels := pgLabels(pg.Name, nil)
labels := reconciler.Labels("proxygroup", pg.Name, "")
labels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal)
ip := fmt.Sprintf("10.0.0.%d", ordinal)
return &corev1.Pod{
cmd/k8s-operator/egress-services.go → k8s-operator/reconciler/egress/services.go
+126 -124
View File
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package egress
import (
"context"
@@ -16,7 +16,7 @@
"reflect"
"slices"
"strings"
"sync"
"time"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
@@ -24,61 +24,105 @@
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/storage/names"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstime"
"tailscale.com/util/clientmetric"
"tailscale.com/util/mak"
"tailscale.com/util/set"
)
const (
reconcilerName = "egress-svcs-reconciler"
reasonEgressSvcInvalid = "EgressSvcInvalid"
reasonEgressSvcValid = "EgressSvcValid"
reasonEgressSvcCreationFailed = "EgressSvcCreationFailed"
reasonProxyGroupNotReady = "ProxyGroupNotReady"
labelProxyGroup = "tailscale.com/proxy-group"
labelSvcType = "tailscale.com/svc-type" // ingress or egress
typeEgress = "egress"
// maxPorts is the maximum number of ports that can be exposed on a
// container. In practice this will be ports in range [10000 - 11000). The
// high range should make it easier to distinguish container ports from
// the tailnet target ports for debugging purposes (i.e when reading
// netfilter rules). The limit of 1000 is somewhat arbitrary, the
// assumption is that this would not be hit in practice.
maxPorts = 1000
indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
tsHealthCheckPortName = "tailscale-health-check"
)
var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
// egressSvcsReconciler reconciles user created ExternalName Services that specify a tailnet
// Reconciler reconciles user created ExternalName Services that specify a tailnet
// endpoint that should be exposed to cluster workloads and an egress ProxyGroup
// on whose proxies it should be exposed.
type egressSvcsReconciler struct {
type Reconciler struct {
client.Client
logger *zap.SugaredLogger
recorder record.EventRecorder
clock tstime.Clock
tsNamespace string
mu sync.Mutex // protects following
svcs set.Slice[types.UID] // UIDs of all currently managed egress Services for ProxyGroup
tracker *reconciler.ResourceTracker
}
// Options contains configuration values shared by the egress reconcilers.
type Options struct {
// Client is used to interact with the Kubernetes API.
Client client.Client
// Recorder is used to emit Kubernetes events. Only Reconciler emits events.
Recorder record.EventRecorder
// TailscaleNamespace is the namespace the operator is installed in, where the ClusterIP Services,
// EndpointSlices and proxy Pods live.
TailscaleNamespace string
// Logger is the logger to use; each reconciler names a child logger after itself.
Logger *zap.SugaredLogger
// Clock is used to stamp condition transitions. Defaults to a real clock when unset.
Clock tstime.Clock
// HTTPClient is used by PodReconciler to call proxy health check endpoints. Defaults to
// http.DefaultClient when unset.
HTTPClient doer
// MaxBackoff caps the backoff between PodReconciler's health check calls. Zero means no cap, which is what
// the operator has always run with; tests set it to keep retries fast.
MaxBackoff time.Duration
}
func (o Options) clock() tstime.Clock {
if o.Clock == nil {
return tstime.DefaultClock{}
}
return o.Clock
}
// NewReconciler returns the reconciler for user-created egress ExternalName Services.
func NewReconciler(opts Options) *Reconciler {
return &Reconciler{
Client: opts.Client,
recorder: opts.Recorder,
logger: opts.Logger.Named(reconcilerName),
clock: opts.clock(),
tsNamespace: opts.TailscaleNamespace,
tracker: reconciler.NewResourceTracker(gaugeEgressServices),
}
}
// Register the Reconciler onto mgr. It watches egress Services directly and egress ProxyGroups so that a ProxyGroup
// becoming ready reconciles every Service exposed on it. It also installs the IndexProxyGroup field index that the
// ProxyGroup handler needs, so that callers can't forget to.
func (esr *Reconciler) Register(mgr manager.Manager) error {
if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), IndexProxyGroup, IndexServices); err != nil {
return fmt.Errorf("failed to set up ProxyGroup indexer for egress Services: %w", err)
}
return builder.
ControllerManagedBy(mgr).
Named(reconcilerName).
Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandler)).
Watches(&tsapi.ProxyGroup{}, handler.EnqueueRequestsFromMapFunc(servicesFromProxyGroup(esr.Client, esr.logger))).
Complete(esr)
}
// Reconcile reconciles an ExternalName Service that specifies a tailnet target and a ProxyGroup on whose proxies should
@@ -99,7 +143,7 @@ type egressSvcsReconciler struct {
//
// - updates the egress service config in a ConfigMap mounted to the ProxyGroup proxies with the tailnet target and the
// portmappings.
func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
func (esr *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
lg := esr.logger.With("Service", req.NamespacedName)
defer lg.Info("reconcile finished")
@@ -139,29 +183,26 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
return res, nil
}
if !slices.Contains(svc.Finalizers, FinalizerName) {
svc.Finalizers = append(svc.Finalizers, FinalizerName)
if !slices.Contains(svc.Finalizers, reconciler.Finalizer) {
svc.Finalizers = append(svc.Finalizers, reconciler.Finalizer)
if err := esr.updateSvcSpec(ctx, svc); err != nil {
err := fmt.Errorf("failed to add finalizer: %w", err)
r := svcConfiguredReason(svc, false, lg)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg)
return res, err
}
esr.mu.Lock()
esr.svcs.Add(svc.UID)
gaugeEgressServices.Set(int64(esr.svcs.Len()))
esr.mu.Unlock()
esr.tracker.Add(svc.UID)
}
if err := esr.maybeCleanupProxyGroupConfig(ctx, svc, lg); err != nil {
err = fmt.Errorf("cleaning up resources for previous ProxyGroup failed: %w", err)
r := svcConfiguredReason(svc, false, lg)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg)
return res, err
}
if err := esr.maybeProvision(ctx, svc, lg); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
if reconciler.IsOptimisticLockError(err) {
lg.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
@@ -171,7 +212,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
return res, nil
}
func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (err error) {
func (esr *Reconciler) maybeProvision(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (err error) {
r := svcConfiguredReason(svc, false, lg)
st := metav1.ConditionFalse
defer func() {
@@ -179,11 +220,11 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1
if st != metav1.ConditionTrue && err != nil {
msg = err.Error()
}
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, lg)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, lg)
}()
crl := egressSvcChildResourceLabels(svc)
clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esr.Client, esr.tsNamespace, crl)
crl := childResourceLabels(svc)
clusterIPSvc, err := reconciler.GetSingleObject[corev1.Service](ctx, esr.Client, esr.tsNamespace, crl)
if err != nil {
err = fmt.Errorf("error retrieving ClusterIP Service: %w", err)
return err
@@ -194,7 +235,7 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1
upToDate := svcConfigurationUpToDate(svc, lg)
provisioned := true
if !upToDate {
if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[AnnotationProxyGroup], svc, clusterIPSvc, lg); err != nil {
if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[reconciler.AnnotationProxyGroup], svc, clusterIPSvc, lg); err != nil {
return err
}
}
@@ -208,7 +249,7 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1
}
// Update ExternalName Service to point at the ClusterIP Service.
clusterDomain := retrieveClusterDomain(esr.tsNamespace, lg)
clusterDomain := reconciler.ClusterDomain(esr.tsNamespace, lg)
clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain)
if svc.Spec.ExternalName != clusterIPSvcFQDN {
lg.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN)
@@ -246,8 +287,8 @@ func addrTypesForClusterIPSvc(clusterIPSvc *corev1.Service) ([]discoveryv1.Addre
// ensureEndpointSlices ensures that EndpointSlices exist for the egress service
// for each IP family supported by the cluster, and that their ports are up to
// date.
func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) error {
crl := egressSvcEpsLabels(svc, clusterIPSvc)
func (esr *Reconciler) ensureEndpointSlices(ctx context.Context, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) error {
crl := epsLabels(svc, clusterIPSvc)
// Only create EndpointSlices for IP families supported by the cluster.
addrTypes, err := addrTypesForClusterIPSvc(clusterIPSvc)
if err != nil {
@@ -263,7 +304,7 @@ func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc,
AddressType: addrType,
Ports: epsPortsFromSvc(clusterIPSvc),
}
if _, err := createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
if _, err := reconciler.CreateOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
e.Labels = eps.Labels
e.AddressType = eps.AddressType
e.Ports = eps.Ports
@@ -277,7 +318,7 @@ func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc,
return nil
}
func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) (*corev1.Service, bool, error) {
func (esr *Reconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) (*corev1.Service, bool, error) {
lg.Infof("updating configuration...")
usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName)
if err != nil {
@@ -347,7 +388,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
})
}
}
var healthCheckPort int32 = defaultLocalAddrPort
var healthCheckPort int32 = tailscaled.HealthCheckPort
for {
if !slices.ContainsFunc(svc.Spec.Ports, func(p corev1.ServicePort) bool {
@@ -363,11 +404,11 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{
Name: tsHealthCheckPortName,
Port: healthCheckPort,
TargetPort: intstr.FromInt(defaultLocalAddrPort),
TargetPort: intstr.FromInt(tailscaled.HealthCheckPort),
Protocol: "TCP",
})
if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) {
if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) {
if clusterIPSvc, err = reconciler.CreateOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) {
svc.Labels = clusterIPSvc.Labels
svc.Spec = clusterIPSvc.Spec
}); err != nil {
@@ -402,7 +443,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
return clusterIPSvc, true, nil
}
func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
func (esr *Reconciler) maybeCleanup(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
logger.Info("ensuring that resources created for egress service are deleted")
// Delete egress service config from the ConfigMap mounted by the proxies.
@@ -416,14 +457,14 @@ func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.S
&corev1.Service{},
&discoveryv1.EndpointSlice{},
}
crl := egressSvcChildResourceLabels(svc)
crl := childResourceLabels(svc)
for _, typ := range types {
if err := esr.DeleteAllOf(ctx, typ, client.InNamespace(esr.tsNamespace), client.MatchingLabels(crl)); err != nil {
return fmt.Errorf("error deleting %s: %w", typ, err)
}
}
ix := slices.Index(svc.Finalizers, FinalizerName)
ix := slices.Index(svc.Finalizers, reconciler.Finalizer)
if ix != -1 {
logger.Debug("Removing Tailscale finalizer from Service")
svc.Finalizers = append(svc.Finalizers[:ix], svc.Finalizers[ix+1:]...)
@@ -431,17 +472,14 @@ func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.S
return fmt.Errorf("failed to remove finalizer: %w", err)
}
}
esr.mu.Lock()
esr.svcs.Remove(svc.UID)
gaugeEgressServices.Set(int64(esr.svcs.Len()))
esr.mu.Unlock()
esr.tracker.Remove(svc.UID)
logger.Info("successfully cleaned up resources for egress Service")
return nil
}
func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) error {
wantsProxyGroup := svc.Annotations[AnnotationProxyGroup]
cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
func (esr *Reconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) error {
wantsProxyGroup := svc.Annotations[reconciler.AnnotationProxyGroup]
cond := reconciler.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
if cond == nil {
return nil
}
@@ -468,7 +506,7 @@ func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Contex
// latest ClusterIP Services via the controller cache. It will not work as well
// once we split into multiple workers- at that point we probably want to set
// used ports on ProxyGroup's status.
func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) (sets.Set[int32], error) {
func (esr *Reconciler) usedPortsForPG(ctx context.Context, pg string) (sets.Set[int32], error) {
svcList := &corev1.ServiceList{}
if err := esr.List(ctx, svcList, client.InNamespace(esr.tsNamespace), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
return nil, fmt.Errorf("error listing Services: %w", err)
@@ -486,10 +524,10 @@ func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string)
// for an egress service exposed on ProxyGroup proxies. The ClusterIP Service
// has no selector. Traffic sent to it will be routed to the endpoints defined
// by an EndpointSlice created for this egress service.
func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
func (esr *Reconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
GenerateName: svcNameBase(crl[LabelParentName]),
GenerateName: svcNameBase(crl[reconciler.LabelParentName]),
Namespace: esr.tsNamespace,
Labels: crl,
},
@@ -500,9 +538,9 @@ func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *c
}
}
func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
crl := egressSvcChildResourceLabels(svc)
cmName := pgEgressCMName(crl[labelProxyGroup])
func (esr *Reconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
crl := childResourceLabels(svc)
cmName := CMName(crl[labelProxyGroup])
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: cmName,
@@ -543,8 +581,8 @@ func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context,
return esr.Update(ctx, cm)
}
func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (bool, error) {
proxyGroupName := svc.Annotations[AnnotationProxyGroup]
func (esr *Reconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (bool, error) {
proxyGroupName := svc.Annotations[reconciler.AnnotationProxyGroup]
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: proxyGroupName,
@@ -552,35 +590,35 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
}
if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) {
lg.Infof("ProxyGroup %q not found, waiting...", proxyGroupName)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg)
reconciler.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil
} else if err != nil {
err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, lg)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, lg)
reconciler.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, err
}
if violations := validateEgressService(svc, pg); len(violations) > 0 {
msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
lg.Info(msg)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, lg)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, lg)
reconciler.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil
}
if !tsoperator.ProxyGroupAvailable(pg) {
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
if !reconciler.ProxyGroupAvailable(pg) {
reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg)
reconciler.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
}
lg.Debugf("egress service is valid")
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, lg)
reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, lg)
return true, nil
}
func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, lg *zap.SugaredLogger) egressservices.Config {
d := retrieveClusterDomain(ns, lg)
d := reconciler.ClusterDomain(ns, lg)
tt := tailnetTargetFromSvc(externalNameSvc)
hep := healthCheckForSvc(clusterIPSvc, d)
cfg := egressservices.Config{
@@ -598,11 +636,11 @@ func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, lg *
}
func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string {
violations := validateService(svc)
violations := reconciler.ValidateService(svc)
// We check that only one of these two is set in the earlier validateService function.
if svc.Annotations[AnnotationTailnetTargetFQDN] == "" && svc.Annotations[AnnotationTailnetTargetIP] == "" {
violations = append(violations, fmt.Sprintf("egress Service for ProxyGroup must have one of %s, %s annotations set", AnnotationTailnetTargetFQDN, AnnotationTailnetTargetIP))
if svc.Annotations[reconciler.AnnotationTailnetTargetFQDN] == "" && svc.Annotations[reconciler.AnnotationTailnetTargetIP] == "" {
violations = append(violations, fmt.Sprintf("egress Service for ProxyGroup must have one of %s, %s annotations set", reconciler.AnnotationTailnetTargetFQDN, reconciler.AnnotationTailnetTargetIP))
}
if len(svc.Spec.Ports) == 0 {
violations = append(violations, "egress Service for ProxyGroup must have at least one target Port specified")
@@ -656,13 +694,13 @@ func unusedPort(usedPorts sets.Set[int32]) int32 {
// Service must contain exactly one of tailscale.com/tailnet-ip,
// tailscale.com/tailnet-fqdn annotations.
func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget {
if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
if fqdn := svc.Annotations[reconciler.AnnotationTailnetTargetFQDN]; fqdn != "" {
return egressservices.TailnetTarget{
FQDN: fqdn,
}
}
return egressservices.TailnetTarget{
IP: svc.Annotations[AnnotationTailnetTargetIP],
IP: svc.Annotations[reconciler.AnnotationTailnetTargetIP],
}
}
@@ -681,13 +719,13 @@ func isEgressSvcForProxyGroup(obj client.Object) bool {
return false
}
annots := s.ObjectMeta.Annotations
return annots[AnnotationProxyGroup] != "" && (annots[AnnotationTailnetTargetFQDN] != "" || annots[AnnotationTailnetTargetIP] != "")
return annots[reconciler.AnnotationProxyGroup] != "" && (annots[reconciler.AnnotationTailnetTargetFQDN] != "" || annots[reconciler.AnnotationTailnetTargetIP] != "")
}
// egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well
// as unmarshalled configuration from the ConfigMap.
func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs egressservices.Configs, err error) {
name := pgEgressCMName(proxyGroupName)
name := CMName(proxyGroupName)
cm = &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: name,
@@ -710,38 +748,8 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
return cm, cfgs, nil
}
// egressSvcChildResourceLabels returns labels that should be applied to the
// ClusterIP Service and the EndpointSlice created for the egress service.
// TODO(irbekrm): we currently set a bunch of labels based on Kubernetes
// resource names (ProxyGroup, Service). Maximum allowed label length is 63
// chars whilst the maximum allowed resource name length is 253 chars, so we
// should probably validate and truncate (?) the names is they are too long.
func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
return map[string]string{
kubetypes.LabelManaged: "true",
LabelParentType: "svc",
LabelParentName: svc.Name,
LabelParentNamespace: svc.Namespace,
labelProxyGroup: svc.Annotations[AnnotationProxyGroup],
labelSvcType: typeEgress,
}
}
// egressEpsLabels returns labels to be added to an EndpointSlice created for an egress service.
func egressSvcEpsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string {
lbels := egressSvcChildResourceLabels(extNSvc)
// Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the
// endpoints defined on this EndpointSlice.
// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
lbels[discoveryv1.LabelServiceName] = clusterIPSvc.Name
// Kubernetes recommends setting this label.
// https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management
lbels[discoveryv1.LabelManagedBy] = "tailscale.com"
return lbels
}
func svcConfigurationUpToDate(svc *corev1.Service, lg *zap.SugaredLogger) bool {
cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
cond := reconciler.GetServiceCondition(svc, tsapi.EgressSvcConfigured)
if cond == nil {
return false
}
@@ -781,23 +789,17 @@ func svcConfiguredReason(svc *corev1.Service, configured bool, lg *zap.SugaredLo
} else {
r = fmt.Sprintf("ConfigurationFailed:%s", r)
}
r += fmt.Sprintf("ProxyGroup:%s", svc.Annotations[AnnotationProxyGroup])
r += fmt.Sprintf("ProxyGroup:%s", svc.Annotations[reconciler.AnnotationProxyGroup])
tt := tailnetTargetFromSvc(svc)
s := cfg{
Ports: svc.Spec.Ports,
TailnetTarget: tt,
ProxyGroup: svc.Annotations[AnnotationProxyGroup],
ProxyGroup: svc.Annotations[reconciler.AnnotationProxyGroup],
}
r += fmt.Sprintf(":Config:%s", cfgHash(s, lg))
return r
}
// tailnetSvc accepts and ExternalName Service name and returns a name that will be used to distinguish this tailnet
// service from other tailnet services exposed to cluster workloads.
func tailnetSvcName(extNSvc *corev1.Service) string {
return fmt.Sprintf("%s-%s", extNSvc.Namespace, extNSvc.Name)
}
// epsPortsFromSvc takes the ClusterIP Service created for an egress service and
// returns its Port array in a form that can be used for an EndpointSlice.
func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) {
@@ -814,7 +816,7 @@ func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) {
// updateSvcSpec ensures that the given Service's spec is updated in cluster, but the local Service object still retains
// the not-yet-applied status.
// TODO(irbekrm): once we do SSA for these patch updates, this will no longer be needed.
func (esr *egressSvcsReconciler) updateSvcSpec(ctx context.Context, svc *corev1.Service) error {
func (esr *Reconciler) updateSvcSpec(ctx context.Context, svc *corev1.Service) error {
st := svc.Status.DeepCopy()
err := esr.Update(ctx, svc)
svc.Status = *st
cmd/k8s-operator/egress-services_test.go → k8s-operator/reconciler/egress/services_test.go
+55 -53
View File
@@ -3,7 +3,7 @@
//go:build !plan9
package main
package egress
import (
"context"
@@ -24,6 +24,8 @@
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/reconcilertest"
"tailscale.com/kube/egressservices"
"tailscale.com/tstest"
"tailscale.com/tstime"
@@ -43,7 +45,7 @@ func TestTailscaleEgressServices(t *testing.T) {
}
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: pgEgressCMName("foo"),
Name: CMName("foo"),
Namespace: "operator-ns",
},
}
@@ -61,12 +63,12 @@ func TestTailscaleEgressServices(t *testing.T) {
}
clock := tstest.NewClock(tstest.ClockOpts{})
esr := &egressSvcsReconciler{
Client: fc,
logger: zl.Sugar(),
clock: clock,
tsNamespace: "operator-ns",
}
esr := NewReconciler(Options{
Client: fc,
Logger: zl.Sugar(),
Clock: clock,
TailscaleNamespace: "operator-ns",
})
tailnetTargetFQDN := "foo.bar.ts.net."
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
@@ -74,8 +76,8 @@ func TestTailscaleEgressServices(t *testing.T) {
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
AnnotationProxyGroup: "foo",
reconciler.AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
reconciler.AnnotationProxyGroup: "foo",
},
},
Spec: corev1.ServiceSpec{
@@ -92,32 +94,32 @@ func TestTailscaleEgressServices(t *testing.T) {
}
t.Run("service_one_unnamed_port", func(t *testing.T) {
mustCreate(t, fc, svc)
expectReconciled(t, esr, "default", "test")
reconcilertest.MustCreate(t, fc, svc)
reconcilertest.ExpectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
t.Run("service_add_two_named_ports", func(t *testing.T) {
svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}}
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
reconcilertest.MustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.Spec.Ports = svc.Spec.Ports
})
expectReconciled(t, esr, "default", "test")
reconcilertest.ExpectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
t.Run("service_add_udp_port", func(t *testing.T) {
svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Port: 53, Protocol: "UDP", Name: "dns"})
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
reconcilertest.MustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.Spec.Ports = svc.Spec.Ports
})
expectReconciled(t, esr, "default", "test")
reconcilertest.ExpectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
t.Run("service_change_protocol", func(t *testing.T) {
svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}, {Port: 53, Protocol: "TCP", Name: "tcp_dns"}}
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
reconcilertest.MustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.Spec.Ports = svc.Spec.Ports
})
expectReconciled(t, esr, "default", "test")
reconcilertest.ExpectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
@@ -134,7 +136,7 @@ func TestTailscaleEgressServices(t *testing.T) {
if err := fc.Delete(t.Context(), eps); err != nil {
t.Fatalf("error deleting EndpointSlice: %v", err)
}
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
@@ -143,23 +145,23 @@ func TestTailscaleEgressServices(t *testing.T) {
if err := fc.Delete(context.Background(), svc); err != nil {
t.Fatalf("error deleting ExternalName Service: %v", err)
}
expectReconciled(t, esr, "default", "test")
reconcilertest.ExpectReconciled(t, esr, "default", "test")
// Verify that ClusterIP Service and EndpointSlice have been deleted.
expectMissing[corev1.Service](t, fc, "operator-ns", name)
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
reconcilertest.ExpectMissing[corev1.Service](t, fc, "operator-ns", name)
reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
// Verify that service config has been deleted from the ConfigMap.
mustNotHaveConfigForSvc(t, fc, svc, cm)
})
}
func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReconciler, svc *corev1.Service, clock *tstest.Clock, zl *zap.Logger, cm *corev1.ConfigMap) {
expectReconciled(t, esr, "default", "test")
func validateReadyService(t *testing.T, fc client.WithWatch, esr *Reconciler, svc *corev1.Service, clock *tstest.Clock, zl *zap.Logger, cm *corev1.ConfigMap) {
reconcilertest.ExpectReconciled(t, esr, "default", "test")
// Verify that a ClusterIP Service has been created.
name := findGenNameForEgressSvcResources(t, fc, svc)
expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc, removeClusterIPsFromSvc)
reconcilertest.ExpectEqual(t, fc, clusterIPSvc(name, svc), reconcilertest.RemoveTargetPorts, reconcilertest.RemoveClusterIPs)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
// Verify that an EndpointSlice has been created.
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4))
reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4))
// Verify that ConfigMap contains configuration for the new egress service.
mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm, zl)
r := svcConfiguredReason(svc, true, zl.Sugar())
@@ -171,7 +173,7 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco
}
svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
expectEqual(t, fc, svc)
reconcilertest.ExpectEqual(t, fc, svc)
}
@@ -179,7 +181,7 @@ func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string
return metav1.Condition{
Type: string(typ),
Status: st,
LastTransitionTime: conditionTime(clock),
LastTransitionTime: reconcilertest.ConditionTime(clock),
Reason: r,
Message: msg,
}
@@ -187,8 +189,8 @@ func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string
func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string {
t.Helper()
labels := egressSvcChildResourceLabels(svc)
s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels)
labels := childResourceLabels(svc)
s, err := reconciler.GetSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels)
if err != nil {
t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err)
}
@@ -199,7 +201,7 @@ func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *c
}
func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
labels := egressSvcChildResourceLabels(extNSvc)
labels := childResourceLabels(extNSvc)
ports := make([]corev1.ServicePort, len(extNSvc.Spec.Ports))
for i, port := range extNSvc.Spec.Ports {
ports[i] = corev1.ServicePort{ // Copy the port to avoid modifying the original.
@@ -246,7 +248,7 @@ func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Se
}
func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service, addrType discoveryv1.AddressType) *discoveryv1.EndpointSlice {
labels := egressSvcChildResourceLabels(extNSvc)
labels := childResourceLabels(extNSvc)
labels[discoveryv1.LabelManagedBy] = "tailscale.com"
labels[discoveryv1.LabelServiceName] = name
suffix := "ipv4"
@@ -335,7 +337,7 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) {
}
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: pgEgressCMName("foo"),
Name: CMName("foo"),
Namespace: "operator-ns",
},
}
@@ -353,20 +355,20 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) {
}
clock := tstest.NewClock(tstest.ClockOpts{})
esr := &egressSvcsReconciler{
Client: fc,
logger: zl.Sugar(),
clock: clock,
tsNamespace: "operator-ns",
}
esr := NewReconciler(Options{
Client: fc,
Logger: zl.Sugar(),
Clock: clock,
TailscaleNamespace: "operator-ns",
})
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetFQDN: "foo.bar.ts.net.",
AnnotationProxyGroup: "foo",
reconciler.AnnotationTailnetTargetFQDN: "foo.bar.ts.net.",
reconciler.AnnotationProxyGroup: "foo",
},
},
Spec: corev1.ServiceSpec{
@@ -383,13 +385,13 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) {
}
t.Run("dual_stack_creates_both_endpoint_slices", func(t *testing.T) {
mustCreate(t, fc, svc)
expectReconciled(t, esr, "default", "test")
reconcilertest.MustCreate(t, fc, svc)
reconcilertest.ExpectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
// Also verify the IPv6 EndpointSlice was created.
name := findGenNameForEgressSvcResources(t, fc, svc)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
})
t.Run("dual_stack_endpointslice_deletion_recovery", func(t *testing.T) {
@@ -406,12 +408,12 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) {
if err := fc.Delete(t.Context(), eps); err != nil {
t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err)
}
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
}
// Reconcile should recreate both.
validateReadyService(t, fc, esr, svc, clock, zl, cm)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
})
t.Run("dual_stack_single_endpointslice_deletion_recovery", func(t *testing.T) {
@@ -427,13 +429,13 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) {
if err := fc.Delete(t.Context(), eps); err != nil {
t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err)
}
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
// Reconcile should recreate the missing IPv6 EndpointSlice while leaving
// the IPv4 one untouched.
validateReadyService(t, fc, esr, svc, clock, zl, cm)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4))
reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4))
})
t.Run("delete_dual_stack_service", func(t *testing.T) {
@@ -441,10 +443,10 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) {
if err := fc.Delete(context.Background(), svc); err != nil {
t.Fatalf("error deleting ExternalName Service: %v", err)
}
expectReconciled(t, esr, "default", "test")
expectMissing[corev1.Service](t, fc, "operator-ns", name)
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv6", name))
reconcilertest.ExpectReconciled(t, esr, "default", "test")
reconcilertest.ExpectMissing[corev1.Service](t, fc, "operator-ns", name)
reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv6", name))
mustNotHaveConfigForSvc(t, fc, svc, cm)
})
}
k8s-operator/reconciler/nameserver/nameserver.go
+1 -2
View File
@@ -27,7 +27,6 @@
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/yaml"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
@@ -132,7 +131,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
oldCnStatus := dnsCfg.Status.DeepCopy()
setStatus := func(dnsCfg *tsapi.DNSConfig, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, r.clock, logger)
reconciler.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, r.clock, logger)
if !apiequality.Semantic.DeepEqual(oldCnStatus, &dnsCfg.Status) {
// An error encountered here should get returned by the Reconcile function.
if updateErr := r.Client.Status().Update(ctx, dnsCfg); updateErr != nil {
k8s-operator/reconciler/peerrelay/peerrelay.go
+11 -6
View File
@@ -31,7 +31,6 @@
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
operatorutils "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
@@ -201,7 +200,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
}
func (r *Reconciler) reportTailnetUnavailable(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, tsErr error) (reconcile.Result, error) {
operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonTailnetUnavailable, tsErr.Error(), r.clock, logger)
setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonTailnetUnavailable, tsErr.Error(), r.clock, logger)
if err := r.Status().Update(ctx, pr); err != nil {
return reconcile.Result{}, errors.Join(tsErr, fmt.Errorf("failed to update PeerRelay status: %w", err))
}
@@ -229,7 +228,7 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, logger *zap.SugaredLogg
// condition so they can fix the spec.
if pr.Spec.AWS != nil && int32(len(pr.Spec.AWS.ElasticIPs)) < replicas {
message := fmt.Sprintf("spec.aws.elasticIPs has %d entries but spec.replicas is %d", len(pr.Spec.AWS.ElasticIPs), replicas)
operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonAWSConfigInvalid, message, r.clock, logger)
setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonAWSConfigInvalid, message, r.clock, logger)
if err := r.Status().Update(ctx, pr); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to update PeerRelay status for %q: %w", pr.Name, err)
}
@@ -358,12 +357,12 @@ func (r *Reconciler) writeStatus(ctx context.Context, logger *zap.SugaredLogger,
switch {
case int32(len(addressed)) < replicas:
message := fmt.Sprintf("%d of %d replicas have a public IP", len(addressed), replicas)
operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonEndpointsPending, message, r.clock, logger)
setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonEndpointsPending, message, r.clock, logger)
case readyReplicas < replicas:
message := fmt.Sprintf("%d of %d pods are ready", readyReplicas, replicas)
operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonPodsPending, message, r.clock, logger)
setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonPodsPending, message, r.clock, logger)
default:
operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionTrue, ReasonReady, ReasonReady, r.clock, logger)
setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionTrue, ReasonReady, ReasonReady, r.clock, logger)
}
if reflect.DeepEqual(prevStatus, &pr.Status) {
@@ -531,3 +530,9 @@ func (r *Reconciler) deleteStatefulSet(ctx context.Context, logger *zap.SugaredL
}
return nil
}
// setCondition sets a condition on pr's status. ObservedGeneration is always the PeerRelay's own generation, so
// callers don't pass it.
func setCondition(pr *tsapi.PeerRelay, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) {
pr.Status.Conditions = reconciler.SetCondition(pr.Status.Conditions, conditionType, status, reason, message, pr.Generation, clock, logger)
}
k8s-operator/reconciler/proxyclass/proxyclass.go
+3 -4
View File
@@ -34,7 +34,6 @@
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/kube/kubetypes"
@@ -167,9 +166,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
if errs := r.validate(ctx, pc, logger); errs != nil {
msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
r.recorder.Event(pc, corev1.EventTypeWarning, ReasonProxyClassInvalid, msg)
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, ReasonProxyClassInvalid, msg, pc.Generation, r.clock, logger)
reconciler.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, ReasonProxyClassInvalid, msg, pc.Generation, r.clock, logger)
} else {
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, ReasonProxyClassValid, ReasonProxyClassValid, pc.Generation, r.clock, logger)
reconciler.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, ReasonProxyClassValid, ReasonProxyClassValid, pc.Generation, r.clock, logger)
}
if !apiequality.Semantic.DeepEqual(oldPCStatus, &pc.Status) {
if err := r.Client.Status().Update(ctx, pc); err != nil {
@@ -480,7 +479,7 @@ func getPortsForProxyClasses(ctx context.Context, c client.Client) (map[string]t
portRanges := make(map[string]tsapi.PortRanges)
for _, i := range pcs.Items {
if !tsoperator.ProxyClassIsReady(&i) {
if !reconciler.ProxyClassIsReady(&i) {
continue
}
if se := i.Spec.StaticEndpoints; se != nil && se.NodePort != nil {
k8s-operator/reconciler/reconciler.go
+359
View File
@@ -9,20 +9,34 @@
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/netip"
"regexp"
"slices"
"strings"
"sync"
"time"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
"tailscale.com/net/dns/resolvconffile"
"tailscale.com/tailcfg"
"tailscale.com/tstime"
"tailscale.com/util/clientmetric"
"tailscale.com/util/dnsname"
"tailscale.com/util/set"
)
@@ -48,6 +62,52 @@
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
)
// Annotations that users set on Services and Ingresses to configure how the operator exposes them. They are read by
// several reconcilers — a Service annotated for egress is acted on by the egress reconcilers, and also by dnsrecords
// when it needs the tailnet target — so they live here rather than in any one reconciler's package.
const (
// AnnotationTailnetTargetIP is the IP of the tailnet node that an egress Service forwards traffic to. Mutually
// exclusive with AnnotationTailnetTargetFQDN.
AnnotationTailnetTargetIP = "tailscale.com/tailnet-ip"
// AnnotationTailnetTargetFQDN is the MagicDNS name of the tailnet node that an egress Service forwards traffic
// to. Mutually exclusive with AnnotationTailnetTargetIP.
AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn"
// AnnotationProxyGroup names the ProxyGroup whose proxies should expose an egress Service.
AnnotationProxyGroup = "tailscale.com/proxy-group"
// AnnotationHostname overrides the tailnet hostname the operator would otherwise derive from a resource's
// namespace and name.
AnnotationHostname = "tailscale.com/hostname"
// AnnotationTags is a comma-separated list of ACL tags to apply to the tailnet device created for a resource.
AnnotationTags = "tailscale.com/tags"
)
// Constants used when determining the cluster's DNS domain; see ClusterDomain.
const (
resolvConfPath = "/etc/resolv.conf"
// DefaultClusterDomain is the cluster domain assumed when it can't be determined from the resolver config. The
// vast majority of clusters use it.
DefaultClusterDomain = "cluster.local"
)
// validMagicDNSName matches a tailnet MagicDNS name, e.g. foo.tail-scale.ts.net.
var validMagicDNSName = regexp.MustCompile(`^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+\.ts\.net\.?$`)
// Reconciler is a CRD reconciler that knows how to register itself with a controller manager, including the watches
// and field indexes it depends on. Every reconciler under this package implements it, so the operator can construct
// them into a slice and register them in a loop rather than wiring each one up by hand.
type Reconciler interface {
reconcile.Reconciler
// Register sets the reconciler up on mgr. Implementations own their own watches and field indexes, so that a
// caller cannot forget to install something the reconciler needs in order to be triggered.
Register(mgr manager.Manager) error
}
// IsOptimisticLockError reports whether err was caused by an optimistic locking conflict, i.e. the object being
// updated was modified after the reconciler read it. Such an error is transient: the reconciler should requeue and
// retry against a freshly read object rather than surface the error.
@@ -301,3 +361,302 @@ func GetSingleObject[T any, O PtrObject[T]](ctx context.Context, c client.Client
}
return ret, nil
}
// IsMagicDNSName reports whether name looks like a tailnet MagicDNS name.
func IsMagicDNSName(name string) bool {
return validMagicDNSName.MatchString(name)
}
// NameForService returns the tailnet hostname to use for svc: the AnnotationHostname override if set, otherwise
// <namespace>-<name>.
func NameForService(svc *corev1.Service) string {
if h, ok := svc.Annotations[AnnotationHostname]; ok {
return h
}
return svc.Namespace + "-" + svc.Name
}
// TagViolations returns a human-readable description of every invalid ACL tag in obj's AnnotationTags, or nil if the
// annotation is absent or every tag is valid.
func TagViolations(obj client.Object) []string {
var violations []string
if obj == nil {
return nil
}
tags, ok := obj.GetAnnotations()[AnnotationTags]
if !ok {
return nil
}
for tag := range strings.SplitSeq(tags, ",") {
tag = strings.TrimSpace(tag)
if err := tailcfg.CheckTag(tag); err != nil {
violations = append(violations, fmt.Sprintf("invalid tag %q: %v", tag, err))
}
}
return violations
}
// ValidateService returns a human-readable description of every way in which svc is not a valid Service for the
// operator to act on, or an empty slice if it is valid. Callers append their own resource-specific violations; the
// checks here are the ones common to every Service the operator exposes.
func ValidateService(svc *corev1.Service) []string {
violations := make([]string, 0)
if svc.Spec.ClusterIP == "None" {
violations = append(violations, "headless Services are not supported.")
}
if svc.Annotations[AnnotationTailnetTargetFQDN] != "" && svc.Annotations[AnnotationTailnetTargetIP] != "" {
violations = append(violations, fmt.Sprintf("only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN))
}
if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
if !IsMagicDNSName(fqdn) {
violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
}
}
if ipStr := svc.Annotations[AnnotationTailnetTargetIP]; ipStr != "" {
ip, err := netip.ParseAddr(ipStr)
if err != nil {
violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q could not be parsed as a valid IP Address, error: %s", AnnotationTailnetTargetIP, ipStr, err))
} else if !ip.IsValid() {
violations = append(violations, fmt.Sprintf("parsed IP address in annotation %s: %q is not valid", AnnotationTailnetTargetIP, ipStr))
}
}
svcName := NameForService(svc)
if err := dnsname.ValidLabel(svcName); err != nil {
if _, ok := svc.Annotations[AnnotationHostname]; ok {
violations = append(violations, fmt.Sprintf("invalid Tailscale hostname specified %q: %s", svcName, err))
} else {
violations = append(violations, fmt.Sprintf("invalid Tailscale hostname %q, use %q annotation to override: %s", svcName, AnnotationHostname, err))
}
}
violations = append(violations, TagViolations(svc)...)
return violations
}
// ClusterDomainOption configures ClusterDomain.
type ClusterDomainOption func(*clusterDomainOpts)
type clusterDomainOpts struct {
resolvConfPath string
}
// WithResolvConfPath overrides the resolver config that ClusterDomain parses. Only tests should need it: in the
// operator it is the resolver config of the Pod the operator itself runs in.
func WithResolvConfPath(path string) ClusterDomainOption {
return func(o *clusterDomainOpts) { o.resolvConfPath = path }
}
// ClusterDomain returns the cluster's DNS domain, determined by parsing the resolver config of the Pod the operator
// runs in. It falls back to DefaultClusterDomain when the config is missing or doesn't have the expected shape, since
// erroring out would be worse than assuming the overwhelmingly common value.
func ClusterDomain(namespace string, logger *zap.SugaredLogger, opts ...ClusterDomainOption) string {
o := clusterDomainOpts{resolvConfPath: resolvConfPath}
for _, opt := range opts {
opt(&o)
}
logger.Infof("attempting to retrieve cluster domain..")
conf, err := resolvconffile.ParseFile(o.resolvConfPath)
if err != nil {
logger.Warnf("error parsing %s to determine cluster domain, defaulting to %q.", o.resolvConfPath, DefaultClusterDomain)
return DefaultClusterDomain
}
return clusterDomainFromResolverConf(conf, namespace, logger)
}
// clusterDomainFromResolverConf attempts to retrieve cluster domain from the provided resolver config.
// It expects the first three search domains in the resolver config to be ['<namespace>.svc.<cluster-domain>, svc.<cluster-domain>, <cluster-domain>, ...]
// If the first three domains match the expected structure, it returns the third.
// If the domains don't match the expected structure or an error is encountered, it defaults to 'cluster.local' domain.
func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string, logger *zap.SugaredLogger) string {
if len(conf.SearchDomains) < 3 {
logger.Warnf(" resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.", len(conf.SearchDomains))
return DefaultClusterDomain
}
first := conf.SearchDomains[0]
if !strings.HasPrefix(string(first), namespace+".svc") {
logger.Warnf("first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.<cluster-domain>")
return DefaultClusterDomain
}
second := conf.SearchDomains[1]
if !strings.HasPrefix(string(second), "svc") {
logger.Warnf("second search domain in resolver config is %s; expected 'svc.<cluster-domain>'.\nDefaulting cluster domain to 'cluster.local'.", second)
return DefaultClusterDomain
}
// Trim the trailing dot for backwards compatibility purposes as the
// cluster domain was previously hardcoded to 'cluster.local' without a
// trailing dot.
probablyClusterDomain := strings.TrimPrefix(second.WithoutTrailingDot(), "svc.")
third := conf.SearchDomains[2]
if !strings.EqualFold(third.WithoutTrailingDot(), probablyClusterDomain) {
logger.Warnf("expected resolver config to contain serch domains <namespace>.svc.<cluster-domain>, svc.<cluster-domain>, <cluster-domain>; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third)
return DefaultClusterDomain
}
logger.Infof("Cluster domain %q extracted from resolver config", probablyClusterDomain)
return probablyClusterDomain
}
// TruncateLabelValue truncates a Kubernetes label value to fit within the
// 63-character limit. If the value exceeds the limit, it is truncated and a
// short hash suffix is appended to preserve uniqueness.
func TruncateLabelValue(val string) string {
const maxLen = 63
if len(val) <= maxLen {
return val
}
hash := sha256.Sum256([]byte(val))
suffix := hex.EncodeToString(hash[:4]) // 8 hex chars
truncated := val[:maxLen-len(suffix)-1]
return truncated + "-" + suffix
}
// SetCondition ensures conds has a condition of the given type with the supplied status, reason, message and observed
// generation, and returns the updated slice. LastTransitionTime is only advanced when the status actually changes, so
// callers can call it on every reconcile without churning the resource; a transition is logged when it happens.
//
// It operates on the condition slice rather than the owning object because the Tailscale CRD types expose
// Status.Conditions as a plain field with no accessor to constrain a type parameter on. Each reconciler wraps this in
// a small helper for its own CRD, e.g.
//
// func setStatus(pr *tsapi.PeerRelay, ct tsapi.ConditionType, st metav1.ConditionStatus, reason, msg string, clock tstime.Clock, lg *zap.SugaredLogger) {
// pr.Status.Conditions = reconciler.SetCondition(pr.Status.Conditions, ct, st, reason, msg, pr.Generation, clock, lg)
// }
func SetCondition(conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []metav1.Condition {
newCondition := metav1.Condition{
Type: string(conditionType),
Status: status,
Reason: reason,
Message: message,
ObservedGeneration: gen,
}
newCondition.LastTransitionTime = metav1.NewTime(clock.Now().Truncate(time.Second))
idx := slices.IndexFunc(conds, func(cond metav1.Condition) bool {
return cond.Type == string(conditionType)
})
if idx == -1 {
return append(conds, newCondition)
}
cond := conds[idx] // update the existing condition
// If this update doesn't contain a state transition, don't update last
// transition time.
if cond.Status == status {
newCondition.LastTransitionTime = cond.LastTransitionTime
} else {
logger.Infof("Status change for condition %s from %s to %s", conditionType, cond.Status, status)
}
conds[idx] = newCondition
return conds
}
// Condition returns the condition of the given type from conds, or nil if it isn't present.
func Condition(conds []metav1.Condition, conditionType tsapi.ConditionType) *metav1.Condition {
idx := slices.IndexFunc(conds, func(cond metav1.Condition) bool {
return cond.Type == string(conditionType)
})
if idx == -1 {
return nil
}
return &conds[idx]
}
// RemoveCondition removes the condition of the given type from conds if present, returning the updated slice.
func RemoveCondition(conds []metav1.Condition, conditionType tsapi.ConditionType) []metav1.Condition {
return slices.DeleteFunc(conds, func(cond metav1.Condition) bool {
return cond.Type == string(conditionType)
})
}
// The helpers below wrap SetCondition and friends for the CRDs whose reconcilers still live in cmd/k8s-operator. As
// each of those moves into its own package under this one it should grow a package-local wrapper instead, at which
// point the corresponding helper here can go.
// SetConnectorCondition ensures that Connector status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
cn.Status.Conditions = SetCondition(cn.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
}
// SetProxyClassCondition ensures that ProxyClass status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetProxyClassCondition(pc *tsapi.ProxyClass, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
pc.Status.Conditions = SetCondition(pc.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
}
// SetDNSConfigCondition ensures that DNSConfig status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes
func SetDNSConfigCondition(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
dnsCfg.Status.Conditions = SetCondition(dnsCfg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
}
// SetServiceCondition ensures that Service status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes. Services carry no meaningful generation for the operator's purposes, so ObservedGeneration is always 0.
func SetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) {
svc.Status.Conditions = SetCondition(svc.Status.Conditions, conditionType, status, reason, message, 0, clock, logger)
}
// GetServiceCondition returns Service condition with the specified type, if it exists on the Service.
func GetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) *metav1.Condition {
return Condition(svc.Status.Conditions, conditionType)
}
// RemoveServiceCondition will remove condition of the given type if it exists.
func RemoveServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) {
svc.Status.Conditions = RemoveCondition(svc.Status.Conditions, conditionType)
}
// SetProxyGroupCondition ensures that ProxyGroup status has a condition with the
// given attributes. LastTransitionTime gets set every time condition's status
// changes.
func SetProxyGroupCondition(pg *tsapi.ProxyGroup, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
pg.Status.Conditions = SetCondition(pg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
}
// ProxyClassIsReady reports whether pc has been validated for its current generation.
func ProxyClassIsReady(pc *tsapi.ProxyClass) bool {
cond := Condition(pc.Status.Conditions, tsapi.ProxyClassReady)
return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pc.Generation
}
// ProxyGroupAvailable reports whether at least one of pg's proxies is running and ready to serve traffic.
func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool {
cond := Condition(pg.Status.Conditions, tsapi.ProxyGroupAvailable)
return cond != nil && cond.Status == metav1.ConditionTrue
}
// ProxyGroupReplicas returns the number of replicas the ProxyGroup asks for, defaulting to two when unset.
func ProxyGroupReplicas(pg *tsapi.ProxyGroup) int32 {
if pg.Spec.Replicas != nil {
return *pg.Spec.Replicas
}
return 2
}
// KubeAPIServerProxyValid reports whether pg's kube-apiserver proxy config has been validated for its current
// generation, and whether the condition is present at all.
func KubeAPIServerProxyValid(pg *tsapi.ProxyGroup) (valid bool, set bool) {
cond := Condition(pg.Status.Conditions, tsapi.KubeAPIServerProxyValid)
return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation, cond != nil
}
// KubeAPIServerProxyConfigured reports whether pg's kube-apiserver proxy has been configured for its current
// generation.
func KubeAPIServerProxyConfigured(pg *tsapi.ProxyGroup) bool {
cond := Condition(pg.Status.Conditions, tsapi.KubeAPIServerProxyConfigured)
return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation
}
// SvcIsReady reports whether the proxy exposing svc is ready.
func SvcIsReady(svc *corev1.Service) bool {
cond := Condition(svc.Status.Conditions, tsapi.ProxyReady)
return cond != nil && cond.Status == metav1.ConditionTrue
}
k8s-operator/reconciler/reconciler_test.go
+239
View File
@@ -9,9 +9,16 @@
"errors"
"fmt"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"go.uber.org/zap"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -20,7 +27,9 @@
"sigs.k8s.io/controller-runtime/pkg/client/fake"
ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/tstest"
"tailscale.com/util/clientmetric"
)
@@ -294,3 +303,233 @@ func TestEnsureAndClearFinalizer(t *testing.T) {
}
})
}
func TestClusterDomain(t *testing.T) {
t.Parallel()
tests := map[string]struct {
// resolvConf is the file ClusterDomain parses. Empty means don't create the file at all, exercising the
// unreadable-config path.
resolvConf string
namespace string
want string
}{
"custom-domain": {
resolvConf: "search foo.svc.department.org.io svc.department.org.io department.org.io\nnameserver 10.96.0.10\n",
namespace: "foo",
want: "department.org.io",
},
"default-domain": {
resolvConf: "search foo.svc.cluster.local svc.cluster.local cluster.local\nnameserver 10.96.0.10\n",
namespace: "foo",
want: "cluster.local",
},
// Everything below should fall back to the default rather than error.
"only-two-search-domains": {
resolvConf: "search svc.department.org.io department.org.io\n",
namespace: "foo",
want: "cluster.local",
},
"first-search-domain-mismatch": {
resolvConf: "search foo.bar.department.org.io svc.department.org.io some.other.fqdn\n",
namespace: "foo",
want: "cluster.local",
},
"second-search-domain-mismatch": {
resolvConf: "search foo.svc.department.org.io foo.department.org.io some.other.fqdn\n",
namespace: "foo",
want: "cluster.local",
},
"third-search-domain-mismatch": {
resolvConf: "search foo.svc.department.org.io svc.department.org.io some.other.fqdn\n",
namespace: "foo",
want: "cluster.local",
},
// The domain here is deliberately not cluster.local: if the namespace check were skipped, this config
// would parse cleanly and yield department.org.io, so the expected fallback distinguishes the two.
"namespace-mismatch": {
resolvConf: "search bar.svc.department.org.io svc.department.org.io department.org.io\n",
namespace: "foo",
want: "cluster.local",
},
"no-search-domains": {
resolvConf: "nameserver 10.96.0.10\n",
namespace: "foo",
want: "cluster.local",
},
"missing-file": {
namespace: "foo",
want: "cluster.local",
},
}
logger := zap.Must(zap.NewDevelopment()).Sugar()
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "resolv.conf")
if tc.resolvConf != "" {
if err := os.WriteFile(path, []byte(tc.resolvConf), 0600); err != nil {
t.Fatalf("writing resolv.conf: %v", err)
}
}
if got := reconciler.ClusterDomain(tc.namespace, logger, reconciler.WithResolvConfPath(path)); got != tc.want {
t.Errorf("ClusterDomain() = %q, want %q", got, tc.want)
}
})
}
}
func TestTruncateLabelValue(t *testing.T) {
tests := []struct {
name string
input string
want string // empty means expect input unchanged
}{
{
name: "short-value-unchanged",
input: "my-service",
},
{
name: "exactly-63-chars-unchanged",
input: strings.Repeat("a", 63),
},
{
name: "64-chars-gets-truncated",
input: strings.Repeat("a", 64),
},
{
name: "very-long-value-gets-truncated",
input: "tailscale-nginx-clickhouse-o11y-server-https-with-extra-long-suffix-that-exceeds-limit",
},
{
name: "253-chars-max-k8s-resource-name",
input: strings.Repeat("x", 253),
},
{
name: "empty-string-unchanged",
input: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := reconciler.TruncateLabelValue(tt.input)
if len(got) > 63 {
t.Errorf("reconciler.TruncateLabelValue(%q) = %q (len %d), exceeds 63 chars", tt.input, got, len(got))
}
if len(tt.input) <= 63 && got != tt.input {
t.Errorf("reconciler.TruncateLabelValue(%q) = %q, want unchanged input", tt.input, got)
}
if len(tt.input) > 63 && got == tt.input {
t.Errorf("reconciler.TruncateLabelValue(%q) was not truncated", tt.input)
}
})
}
}
func TestTruncateLabelValueDeterministic(t *testing.T) {
input := strings.Repeat("a", 100)
first := reconciler.TruncateLabelValue(input)
for range 10 {
got := reconciler.TruncateLabelValue(input)
if got != first {
t.Fatalf("non-deterministic: got %q, want %q", got, first)
}
}
}
func TestTruncateLabelValueUniqueness(t *testing.T) {
// Two inputs sharing a long prefix but differing at the end should produce different outputs.
a := strings.Repeat("a", 100) + "-one"
b := strings.Repeat("a", 100) + "-two"
if reconciler.TruncateLabelValue(a) == reconciler.TruncateLabelValue(b) {
t.Errorf("collision: %q and %q produce the same truncated label", a, b)
}
}
func TestSetConnectorCondition(t *testing.T) {
cn := tsapi.Connector{}
clock := tstest.NewClock(tstest.ClockOpts{})
fakeNow := metav1.NewTime(clock.Now().Truncate(time.Second))
fakePast := metav1.NewTime(clock.Now().Truncate(time.Second).Add(-5 * time.Minute))
zl, err := zap.NewDevelopment()
assert.Nil(t, err)
// Set up a new condition
reconciler.SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "someReason", "someMsg", 1, clock, zl.Sugar())
assert.Equal(t, cn, tsapi.Connector{
Status: tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
Reason: "someReason",
Message: "someMsg",
ObservedGeneration: 1,
LastTransitionTime: fakeNow,
},
},
},
})
// Modify status of an existing condition
cn.Status = tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionFalse,
Reason: "someReason",
Message: "someMsg",
ObservedGeneration: 1,
LastTransitionTime: fakePast,
},
},
}
reconciler.SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar())
assert.Equal(t, cn, tsapi.Connector{
Status: tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
Reason: "anotherReason",
Message: "anotherMsg",
ObservedGeneration: 2,
LastTransitionTime: fakeNow,
},
},
},
})
// Don't modify last transition time if status hasn't changed
cn.Status = tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
Reason: "someReason",
Message: "someMsg",
ObservedGeneration: 1,
LastTransitionTime: fakePast,
},
},
}
reconciler.SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar())
assert.Equal(t, cn, tsapi.Connector{
Status: tsapi.ConnectorStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
Reason: "anotherReason",
Message: "anotherMsg",
ObservedGeneration: 2,
LastTransitionTime: fakePast,
},
},
},
})
}
k8s-operator/reconciler/reconcilertest/conditions.go
+8
View File
@@ -7,11 +7,13 @@
import (
"testing"
"time"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tstime"
)
// The condition helpers below take a []metav1.Condition rather than the owning object. Every Tailscale CRD stores its
@@ -57,3 +59,9 @@ func ExpectConditionStatus(t *testing.T, conds []metav1.Condition, conditionType
conditionType, got.Status, got.Reason, status, reason)
}
}
// ConditionTime returns the LastTransitionTime a reconciler using clock would stamp on a condition. The operator
// truncates to the second, so a test building an expected condition has to do the same.
func ConditionTime(clock tstime.Clock) metav1.Time {
return metav1.NewTime(clock.Now().Truncate(time.Second))
}
k8s-operator/reconciler/reconcilertest/reconcilertest.go
+47
View File
@@ -24,6 +24,7 @@
"time"
"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
@@ -78,6 +79,14 @@ func MustCreate(t *testing.T, c client.Client, obj client.Object) {
}
}
// MustCreateAll creates each of objs in order.
func MustCreateAll(t *testing.T, c client.Client, objs ...client.Object) {
t.Helper()
for _, obj := range objs {
MustCreate(t, c, obj)
}
}
// MustGet reads ns/name into obj, failing the test if it is absent. Use ExpectMissing to assert the opposite.
func MustGet(t *testing.T, c client.Client, ns, name string, obj client.Object) {
t.Helper()
@@ -176,6 +185,27 @@ func ExpectReconciled(t *testing.T, r reconcile.Reconciler, ns, name string) {
}
}
// ExpectRequeue runs a single reconcile for ns/name and asserts it succeeded but asked to be retried later, i.e. the
// reconciler is waiting on state it doesn't control.
func ExpectRequeue(t *testing.T, r reconcile.Reconciler, ns, name string) {
t.Helper()
res, err := r.Reconcile(t.Context(), request(ns, name))
if err != nil {
t.Fatalf("Reconcile: unexpected error: %v", err)
}
if res.RequeueAfter == 0 {
t.Fatalf("expected timed requeue, got success")
}
}
// ExpectReconcileError runs a single reconcile for ns/name and asserts it returned an error.
func ExpectReconcileError(t *testing.T, r reconcile.Reconciler, ns, name string) {
t.Helper()
if _, err := r.Reconcile(t.Context(), request(ns, name)); err == nil {
t.Error("Reconcile: expected error but did not get one")
}
}
func request(ns, name string) reconcile.Request {
return reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: name}}
}
@@ -200,3 +230,20 @@ func ExpectEvents(t *testing.T, rec *record.FakeRecorder, want []string) {
}
}
}
// RemoveTargetPorts is an ExpectEqual modifier that blanks the TargetPort of every port on a Service. Reconcilers that
// allocate a target port pick it at random, so a test can assert on the rest of the Service without pinning it.
func RemoveTargetPorts(svc *corev1.Service) {
ports := make([]corev1.ServicePort, 0, len(svc.Spec.Ports))
for _, p := range svc.Spec.Ports {
ports = append(ports, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port, Name: p.Name})
}
svc.Spec.Ports = ports
}
// RemoveClusterIPs is an ExpectEqual modifier that blanks a Service's ClusterIPs, which are assigned by the cluster
// rather than by the reconciler under test.
func RemoveClusterIPs(svc *corev1.Service) {
svc.Spec.ClusterIP = ""
svc.Spec.ClusterIPs = nil
}
k8s-operator/reconciler/recorder/recorder.go
+1 -2
View File
@@ -33,7 +33,6 @@
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/reconciler/tailscaled"
@@ -158,7 +157,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
oldTSRStatus := tsr.Status.DeepCopy()
setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger)
tsr.Status.Conditions = reconciler.SetCondition(tsr.Status.Conditions, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger)
if !apiequality.Semantic.DeepEqual(oldTSRStatus, &tsr.Status) {
// An error encountered here should get returned by the Reconcile function.
if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil {
k8s-operator/reconciler/tailnet/tailnet.go
+10 -5
View File
@@ -28,7 +28,6 @@
"tailscale.com/client/tailscale/v2"
"tailscale.com/ipn"
operatorutils "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/k8s-operator/reconciler"
"tailscale.com/k8s-operator/tsclient"
@@ -165,7 +164,7 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet)
// The referenced Secret does not exist within the tailscale namespace, so we'll mark the Tailnet as not ready
// for use.
if apierrors.IsNotFound(err) {
operatorutils.SetTailnetCondition(
setCondition(
tailnet,
tsapi.TailnetReady,
metav1.ConditionFalse,
@@ -215,7 +214,7 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet)
return reconcile.Result{RequeueAfter: time.Minute / 2}, nil
}
operatorutils.SetTailnetCondition(
setCondition(
tailnet,
tsapi.TailnetReady,
metav1.ConditionTrue,
@@ -334,7 +333,7 @@ func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient tsclient.Cl
}
if errs != nil {
operatorutils.SetTailnetCondition(
setCondition(
tailnet,
tsapi.TailnetReady,
metav1.ConditionFalse,
@@ -366,7 +365,7 @@ func (r *Reconciler) ensureSecret(tailnet *tsapi.Tailnet, secret *corev1.Secret)
return true
}
operatorutils.SetTailnetCondition(
setCondition(
tailnet,
tsapi.TailnetReady,
metav1.ConditionFalse,
@@ -378,3 +377,9 @@ func (r *Reconciler) ensureSecret(tailnet *tsapi.Tailnet, secret *corev1.Secret)
return false
}
// setCondition sets a condition on tn's status. ObservedGeneration is always the Tailnet's own generation, so callers
// don't pass it.
func setCondition(tn *tsapi.Tailnet, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) {
tn.Status.Conditions = reconciler.SetCondition(tn.Status.Conditions, conditionType, status, reason, message, tn.Generation, clock, logger)
}
k8s-operator/utils.go
-16
View File
@@ -7,8 +7,6 @@
package kube
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/netip"
"strconv"
@@ -94,17 +92,3 @@ func ResolveViaDomain(name string) (netip.Addr, bool) {
out, _ := tsaddr.MapVia(uint32(prefix), netip.PrefixFrom(ip4, ip4.BitLen()))
return out.Addr(), true
}
// TruncateLabelValue truncates a Kubernetes label value to fit within the
// 63-character limit. If the value exceeds the limit, it is truncated and a
// short hash suffix is appended to preserve uniqueness.
func TruncateLabelValue(val string) string {
const maxLen = 63
if len(val) <= maxLen {
return val
}
hash := sha256.Sum256([]byte(val))
suffix := hex.EncodeToString(hash[:4]) // 8 hex chars
truncated := val[:maxLen-len(suffix)-1]
return truncated + "-" + suffix
}
k8s-operator/utils_test.go
+73 -61
View File
@@ -3,76 +3,88 @@
//go:build !plan9
package kube
package kube_test
import (
"strings"
"fmt"
"net/netip"
"testing"
kube "tailscale.com/k8s-operator"
"tailscale.com/tailcfg"
)
func TestTruncateLabelValue(t *testing.T) {
tests := []struct {
name string
input string
want string // empty means expect input unchanged
}{
{
name: "short-value-unchanged",
input: "my-service",
},
{
name: "exactly-63-chars-unchanged",
input: strings.Repeat("a", 63),
},
{
name: "64-chars-gets-truncated",
input: strings.Repeat("a", 64),
},
{
name: "very-long-value-gets-truncated",
input: "tailscale-nginx-clickhouse-o11y-server-https-with-extra-long-suffix-that-exceeds-limit",
},
{
name: "253-chars-max-k8s-resource-name",
input: strings.Repeat("x", 253),
},
{
name: "empty-string-unchanged",
input: "",
},
// TestTailscaledConfigFileNameRoundTrip pins the config file naming contract. The operator writes these names into a
// Secret and containerboot parses them back out, so the two functions must stay inverses of each other and the format
// must not drift.
func TestTailscaledConfigFileNameRoundTrip(t *testing.T) {
t.Parallel()
for _, capVer := range []tailcfg.CapabilityVersion{0, 1, 95, 106, 32767} {
name := kube.TailscaledConfigFileName(capVer)
if want := fmt.Sprintf("cap-%d.hujson", capVer); name != want {
t.Errorf("TailscaledConfigFileName(%d) = %q, want %q", capVer, name, want)
}
got, err := kube.CapVerFromFileName(name)
if err != nil {
t.Errorf("CapVerFromFileName(%q): %v", name, err)
continue
}
if got != capVer {
t.Errorf("round trip of %d gave %d", capVer, got)
}
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := TruncateLabelValue(tt.input)
if len(got) > 63 {
t.Errorf("TruncateLabelValue(%q) = %q (len %d), exceeds 63 chars", tt.input, got, len(got))
// Pre-config-file proxies wrote a file simply named "tailscaled"; it maps to capability version 0.
got, err := kube.CapVerFromFileName("tailscaled")
if err != nil {
t.Errorf(`CapVerFromFileName("tailscaled"): %v`, err)
}
if got != 0 {
t.Errorf(`CapVerFromFileName("tailscaled") = %d, want 0`, got)
}
if _, err := kube.CapVerFromFileName("not-a-config-file"); err == nil {
t.Error("CapVerFromFileName(\"not-a-config-file\") succeeded, want error")
}
}
func TestResolveViaDomain(t *testing.T) {
t.Parallel()
tests := map[string]struct {
name string
want string // empty means expect ok == false
}{
"bare": {name: "10-1-2-3-via-7", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"},
"trailing-dot": {name: "10-1-2-3-via-7.", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"},
"ts.net-domain": {name: "10-1-2-3-via-7.foo.ts.net.", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"},
"tailscale.net-domain": {name: "10-1-2-3-via-7.foo.tailscale.net", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"},
"hex-site-id": {name: "10-1-2-3-via-0x7", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"},
"too-short": {name: "0-0-0-via"},
"no-via": {name: "10-1-2-3-7.foo.ts.net"},
"foreign-domain": {name: "10-1-2-3-via-7.example.com"},
"not-an-ipv4": {name: "10-1-2-via-7"},
"site-id-not-a-number": {name: "10-1-2-3-via-abc"},
}
for tn, tc := range tests {
t.Run(tn, func(t *testing.T) {
t.Parallel()
got, ok := kube.ResolveViaDomain(tc.name)
if tc.want == "" {
if ok {
t.Fatalf("ResolveViaDomain(%q) = %v, true; want ok == false", tc.name, got)
}
return
}
if len(tt.input) <= 63 && got != tt.input {
t.Errorf("TruncateLabelValue(%q) = %q, want unchanged input", tt.input, got)
if !ok {
t.Fatalf("ResolveViaDomain(%q) returned ok == false, want %s", tc.name, tc.want)
}
if len(tt.input) > 63 && got == tt.input {
t.Errorf("TruncateLabelValue(%q) was not truncated", tt.input)
if got != netip.MustParseAddr(tc.want) {
t.Errorf("ResolveViaDomain(%q) = %v, want %v", tc.name, got, tc.want)
}
})
}
}
func TestTruncateLabelValueDeterministic(t *testing.T) {
input := strings.Repeat("a", 100)
first := TruncateLabelValue(input)
for range 10 {
got := TruncateLabelValue(input)
if got != first {
t.Fatalf("non-deterministic: got %q, want %q", got, first)
}
}
}
func TestTruncateLabelValueUniqueness(t *testing.T) {
// Two inputs sharing a long prefix but differing at the end should produce different outputs.
a := strings.Repeat("a", 100) + "-one"
b := strings.Repeat("a", 100) + "-two"
if TruncateLabelValue(a) == TruncateLabelValue(b) {
t.Errorf("collision: %q and %q produce the same truncated label", a, b)
}
}