diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go index 37260c7a0..3392cf8e8 100644 --- a/cmd/k8s-operator/api-server-proxy-pg.go +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -25,8 +25,8 @@ "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/client/tailscale/v2" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" @@ -123,7 +123,7 @@ func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, s // Update the condition based on how many pods are advertising the service conditionStatus := metav1.ConditionFalse conditionReason := reasonKubeAPIServerProxyNoBackends - conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, pgReplicas(pg)) + conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, reconciler.ProxyGroupReplicas(pg)) pg.Status.URL = "" if podsAdvertising > 0 { @@ -135,7 +135,7 @@ func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, s } } - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger) if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { // An error encountered here should get returned by the Reconcile function. @@ -143,7 +143,7 @@ func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, s } }() - if !tsoperator.ProxyGroupAvailable(pg) { + if !reconciler.ProxyGroupAvailable(pg) { return nil } @@ -172,12 +172,12 @@ func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, s msg := fmt.Sprintf("error ensuring exclusive ownership of Tailscale Service %s: %v. %s", serviceName, err, instr) logger.Warn(msg) r.recorder.Event(pg, corev1.EventTypeWarning, "InvalidTailscaleService", msg) - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger) return nil } // After getting this far, we know the Tailscale Service is valid. - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger) // Service tags are limited to matching the ProxyGroup's tags until we have // support for querying peer caps for a Service-bound request. diff --git a/cmd/k8s-operator/api-server-proxy-pg_test.go b/cmd/k8s-operator/api-server-proxy-pg_test.go index 131b7b823..b6438ad89 100644 --- a/cmd/k8s-operator/api-server-proxy-pg_test.go +++ b/cmd/k8s-operator/api-server-proxy-pg_test.go @@ -18,8 +18,8 @@ "sigs.k8s.io/controller-runtime/pkg/client/fake" "tailscale.com/client/tailscale/v2" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/k8s-proxy/conf" @@ -130,8 +130,8 @@ func TestAPIServerProxyReconciler(t *testing.T) { } expectReconciled(t, r, "", pgName) pg.ObjectMeta.Finalizers = []string{proxyPGFinalizerName} - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger) - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) expectEqual(t, fc, pg, omitPGStatusConditionMessages) expectMissing[corev1.Secret](t, fc, ns, defaultDomain) expectMissing[rbacv1.Role](t, fc, ns, defaultDomain) @@ -178,8 +178,8 @@ func TestAPIServerProxyReconciler(t *testing.T) { if !reflect.DeepEqual(tsSvc, expectedTSSvc) { t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc) } - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger) - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) expectEqual(t, fc, pg, omitPGStatusConditionMessages) expectedCfg.APIServerProxy.ServiceName = new(tailcfg.ServiceName("svc:" + pgName)) @@ -213,7 +213,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { }) expectReconciled(t, r, "", pgName) - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) pg.Status.URL = "https://" + defaultDomain expectEqual(t, fc, pg, omitPGStatusConditionMessages) @@ -243,7 +243,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { expectedCfg.APIServerProxy.ServiceName = new(updatedServiceName) expectedCfg.AdvertiseServices = nil expectCfg(&expectedCfg) - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) pg.Status.URL = "" expectEqual(t, fc, pg, omitPGStatusConditionMessages) @@ -262,7 +262,7 @@ func TestAPIServerProxyReconciler(t *testing.T) { expectReconciled(t, r, "", pgName) expectedCfg.AdvertiseServices = []string{updatedServiceName.String()} expectCfg(&expectedCfg) - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) pg.Status.URL = "https://" + updatedDomain // Delete the ProxyGroup and verify Tailscale Service and cert resources are cleaned up. diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index 323dc7b86..ebc2a536e 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -26,8 +26,8 @@ "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" @@ -115,7 +115,7 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque oldCnStatus := cn.Status.DeepCopy() setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger) + reconciler.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger) var updateErr error if !apiequality.Semantic.DeepEqual(oldCnStatus, &cn.Status) { // An error encountered here should get returned by the Reconcile function. diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index a054db37b..ae9232c78 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -353,7 +353,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/apiserver/pkg/authentication/user from k8s.io/apiserver/pkg/endpoints/request k8s.io/apiserver/pkg/endpoints/request from tailscale.com/k8s-operator/api-proxy k8s.io/apiserver/pkg/features from k8s.io/apiserver/pkg/endpoints/request - k8s.io/apiserver/pkg/storage/names from tailscale.com/cmd/k8s-operator + k8s.io/apiserver/pkg/storage/names from tailscale.com/cmd/k8s-operator+ k8s.io/apiserver/pkg/util/feature from k8s.io/apiserver/pkg/endpoints/request+ k8s.io/client-go/applyconfigurations/admissionregistration/v1 from k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1+ k8s.io/client-go/applyconfigurations/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1 @@ -769,6 +769,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+ tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet+ tailscale.com/k8s-operator/reconciler/dnsrecords from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator/reconciler/egress from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/nameserver from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/peerrelay from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/proxyclass from tailscale.com/cmd/k8s-operator @@ -781,7 +782,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/sessionrecording/tsrecorder from tailscale.com/k8s-operator/sessionrecording+ tailscale.com/k8s-operator/sessionrecording/ws from tailscale.com/k8s-operator/sessionrecording tailscale.com/k8s-operator/tsclient from tailscale.com/cmd/k8s-operator+ - tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator + tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator+ tailscale.com/kube/ingressservices from tailscale.com/cmd/k8s-operator tailscale.com/kube/k8s-proxy/conf from tailscale.com/cmd/k8s-operator tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ @@ -798,7 +799,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ 💣 tailscale.com/net/batching from tailscale.com/wgengine/magicsock tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+ tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ - tailscale.com/net/dns/resolvconffile from tailscale.com/cmd/k8s-operator+ + tailscale.com/net/dns/resolvconffile from tailscale.com/k8s-operator/reconciler+ tailscale.com/net/dns/resolver from tailscale.com/net/dns+ tailscale.com/net/dnscache from tailscale.com/control/controlclient+ tailscale.com/net/dnsfallback from tailscale.com/control/controlclient+ @@ -879,7 +880,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/types/structs from tailscale.com/control/controlclient+ tailscale.com/types/tkatype from tailscale.com/client/local+ tailscale.com/types/views from tailscale.com/appc+ - tailscale.com/util/backoff from tailscale.com/cmd/k8s-operator+ + tailscale.com/util/backoff from tailscale.com/control/controlclient+ tailscale.com/util/bufiox from tailscale.com/types/key tailscale.com/util/checkchange from tailscale.com/ipn/ipnlocal+ tailscale.com/util/cibuild from tailscale.com/health+ @@ -959,7 +960,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ golang.org/x/exp/constraints from tailscale.com/tsweb/varz+ golang.org/x/exp/maps from tailscale.com/ipn/store/mem+ - golang.org/x/exp/slices from tailscale.com/cmd/k8s-operator+ + golang.org/x/exp/slices from tailscale.com/cmd/k8s-operator golang.org/x/net/bpf from github.com/mdlayher/netlink+ golang.org/x/net/dns/dnsmessage from tailscale.com/appc+ golang.org/x/net/http/httpguts from golang.org/x/net/http2+ diff --git a/cmd/k8s-operator/e2e/egress_test.go b/cmd/k8s-operator/e2e/egress_test.go index 9f0a530ce..bb3b866b7 100644 --- a/cmd/k8s-operator/e2e/egress_test.go +++ b/cmd/k8s-operator/e2e/egress_test.go @@ -11,8 +11,8 @@ corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/tstest" ) @@ -33,7 +33,7 @@ func TestEgress(t *testing.T) { "tailscale.com/tailnet-ip": tnTarget.ipv4, }) createAndCleanup(t, kubeClient, svc) - waitForEgress(t, svc.Name, kube.SvcIsReady) + waitForEgress(t, svc.Name, reconciler.SvcIsReady) testEgressIsReachable(t, ns, svc.Name) }) @@ -45,7 +45,7 @@ func TestEgress(t *testing.T) { "tailscale.com/tailnet-ip": tnTarget.ipv6, }) createAndCleanup(t, kubeClient, svc) - waitForEgress(t, svc.Name, kube.SvcIsReady) + waitForEgress(t, svc.Name, reconciler.SvcIsReady) testEgressIsReachable(t, ns, svc.Name) }) @@ -54,7 +54,7 @@ func TestEgress(t *testing.T) { "tailscale.com/tailnet-fqdn": tnTarget.fqdn, }) createAndCleanup(t, kubeClient, svc) - waitForEgress(t, svc.Name, kube.SvcIsReady) + waitForEgress(t, svc.Name, reconciler.SvcIsReady) testEgressIsReachable(t, ns, svc.Name) }) } @@ -165,7 +165,7 @@ func egressService(name string, annotations map[string]string) *corev1.Service { } func pgEgressReady(svc *corev1.Service) bool { - cond := kube.GetServiceCondition(svc, tsapi.EgressSvcReady) + cond := reconciler.GetServiceCondition(svc, tsapi.EgressSvcReady) return cond != nil && cond.Status == metav1.ConditionTrue } diff --git a/cmd/k8s-operator/e2e/ingress_test.go b/cmd/k8s-operator/e2e/ingress_test.go index 67dfd1b0b..8772c5196 100644 --- a/cmd/k8s-operator/e2e/ingress_test.go +++ b/cmd/k8s-operator/e2e/ingress_test.go @@ -18,8 +18,8 @@ "sigs.k8s.io/controller-runtime/pkg/client" "tailscale.com/client/tailscale/v2" - kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/util/httpm" @@ -64,7 +64,7 @@ func TestL3Ingress(t *testing.T) { if err := get(t.Context(), kubeClient, maybeReadySvc); err != nil { return err } - isReady := kube.SvcIsReady(maybeReadySvc) + isReady := reconciler.SvcIsReady(maybeReadySvc) if isReady { t.Log("Service is ready") return nil diff --git a/cmd/k8s-operator/e2e/setup.go b/cmd/k8s-operator/e2e/setup.go index 93289eab4..6829c3b50 100644 --- a/cmd/k8s-operator/e2e/setup.go +++ b/cmd/k8s-operator/e2e/setup.go @@ -61,8 +61,8 @@ "tailscale.com/client/tailscale/v2" "tailscale.com/ipn" "tailscale.com/ipn/store/mem" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/tsnet" "tailscale.com/util/must" ) @@ -729,7 +729,7 @@ func applyDefaultProxyClass(ctx context.Context, logger *zap.SugaredLogger, cl c if err := cl.Get(ctx, client.ObjectKeyFromObject(pc), pc); err != nil { return fmt.Errorf("failed to get default ProxyClass: %w", err) } - if tsoperator.ProxyClassIsReady(pc) { + if reconciler.ProxyClassIsReady(pc) { break } logger.Info("waiting for default ProxyClass to be ready...") diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 33291e3d0..765d7bbc7 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -32,8 +32,8 @@ "tailscale.com/client/tailscale/v2" "tailscale.com/ipn" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" @@ -182,7 +182,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } logger = logger.With("ProxyGroup", pgName) - if !tsoperator.ProxyGroupAvailable(pg) { + if !reconciler.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") return false, nil } @@ -679,7 +679,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki var errs []error // Validate tags if present - violations := tagViolations(ing) + violations := reconciler.TagViolations(ing) if len(violations) > 0 { errs = append(errs, fmt.Errorf("Ingress contains invalid tags: %v", strings.Join(violations, ","))) } @@ -702,7 +702,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki } // Validate ProxyGroup readiness - if !tsoperator.ProxyGroupAvailable(pg) { + if !reconciler.ProxyGroupAvailable(pg) { errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name)) } @@ -1108,7 +1108,7 @@ func certResourceLabels(pgName, domain string) map[string]string { return map[string]string{ kubetypes.LabelManaged: "true", labelProxyGroup: pgName, - labelDomain: tsoperator.TruncateLabelValue(domain), + labelDomain: reconciler.TruncateLabelValue(domain), } } @@ -1135,22 +1135,3 @@ func hasCerts(ctx context.Context, cl client.Client, ns string, svc tailcfg.Serv return len(cert) > 0 && len(key) > 0, nil } - -func tagViolations(obj client.Object) []string { - var violations []string - if obj == nil { - return nil - } - tags, ok := obj.GetAnnotations()[AnnotationTags] - if !ok { - return nil - } - - for tag := range strings.SplitSeq(tags, ",") { - tag = strings.TrimSpace(tag) - if err := tailcfg.CheckTag(tag); err != nil { - violations = append(violations, fmt.Sprintf("invalid tag %q: %v", tag, err)) - } - } - return violations -} diff --git a/cmd/k8s-operator/metrics_resources.go b/cmd/k8s-operator/metrics_resources.go index b9f2d6df6..780307ce8 100644 --- a/cmd/k8s-operator/metrics_resources.go +++ b/cmd/k8s-operator/metrics_resources.go @@ -20,8 +20,8 @@ "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - kube "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" ) @@ -239,13 +239,13 @@ func metricsResourceLabels(opts *metricsOpts) map[string]string { kubetypes.LabelManaged: "true", labelMetricsTarget: opts.proxyStsName, labelPromProxyType: opts.proxyType, - labelPromProxyParentName: kube.TruncateLabelValue(opts.proxyLabels[LabelParentName]), + labelPromProxyParentName: reconciler.TruncateLabelValue(opts.proxyLabels[LabelParentName]), } // Include namespace label for proxies created for a namespaced type. if isNamespacedProxyType(opts.proxyType) { - lbls[labelPromProxyParentNamespace] = kube.TruncateLabelValue(opts.proxyLabels[LabelParentNamespace]) + lbls[labelPromProxyParentNamespace] = reconciler.TruncateLabelValue(opts.proxyLabels[LabelParentNamespace]) } - lbls[labelPromJob] = kube.TruncateLabelValue(promJobName(opts)) + lbls[labelPromJob] = reconciler.TruncateLabelValue(promJobName(opts)) return lbls } @@ -262,11 +262,11 @@ func promJobName(opts *metricsOpts) string { func metricsSvcSelector(proxyLabels map[string]string, proxyType string) map[string]string { sel := map[string]string{ labelPromProxyType: proxyType, - labelPromProxyParentName: kube.TruncateLabelValue(proxyLabels[LabelParentName]), + labelPromProxyParentName: reconciler.TruncateLabelValue(proxyLabels[LabelParentName]), } // Include namespace label for proxies created for a namespaced type. if isNamespacedProxyType(proxyType) { - sel[labelPromProxyParentNamespace] = kube.TruncateLabelValue(proxyLabels[LabelParentNamespace]) + sel[labelPromProxyParentNamespace] = reconciler.TruncateLabelValue(proxyLabels[LabelParentNamespace]) } return sel } diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index a084874cb..2bafbf79c 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -10,9 +10,7 @@ import ( "context" "fmt" - "net/http" "os" - "regexp" "strconv" "strings" "time" @@ -54,7 +52,9 @@ "tailscale.com/ipn/store/kubestore" apiproxy "tailscale.com/k8s-operator/api-proxy" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/dnsrecords" + "tailscale.com/k8s-operator/reconciler/egress" "tailscale.com/k8s-operator/reconciler/nameserver" "tailscale.com/k8s-operator/reconciler/peerrelay" "tailscale.com/k8s-operator/reconciler/proxyclass" @@ -354,39 +354,82 @@ func runReconcilers(opts reconcilerOpts) { } clients := tsclient.NewProvider(tsclient.Wrap(opts.tsClient)) + eventRecorder := mgr.GetEventRecorderFor("tailscale-operator") - tailnetOptions := tailnet.ReconcilerOptions{ + // The four egress reconcilers cooperate on the same resources and so share one set of options. + egressOptions := egress.Options{ Client: mgr.GetClient(), + Recorder: eventRecorder, TailscaleNamespace: opts.tailscaleNamespace, - OperatorSAName: opts.operatorSAName, + Logger: opts.log, Clock: tstime.DefaultClock{}, - Logger: opts.log, - Registry: clients, } - if err = tailnet.NewReconciler(tailnetOptions).Register(mgr); err != nil { - startlog.Fatalf("could not register tailnet reconciler: %v", err) + // Reconcilers that live in their own packages under k8s-operator/reconciler register themselves, including the + // watches and field indexes they depend on, so they can all be set up in one loop. Registration order carries no + // meaning to controller-runtime. The reconcilers still defined in this package are wired up by hand below. + // + // TODO (irbekrm): switch to metadata-only watches for resources whose + // spec we don't need to inspect to reduce memory consumption. + // https://github.com/kubernetes-sigs/controller-runtime/issues/1159 + reconcilers := []reconciler.Reconciler{ + tailnet.NewReconciler(tailnet.ReconcilerOptions{ + Client: mgr.GetClient(), + TailscaleNamespace: opts.tailscaleNamespace, + OperatorSAName: opts.operatorSAName, + Clock: tstime.DefaultClock{}, + Logger: opts.log, + Registry: clients, + }), + proxygrouppolicy.NewReconciler(proxygrouppolicy.ReconcilerOptions{ + Client: mgr.GetClient(), + }), + peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: mgr.GetClient(), + TailscaleNamespace: opts.tailscaleNamespace, + ProxyImage: opts.proxyImage, + DefaultTags: strings.Split(opts.proxyTags, ","), + Clients: clients, + Logger: opts.log, + }), + nameserver.NewReconciler(nameserver.ReconcilerOptions{ + Client: mgr.GetClient(), + Recorder: eventRecorder, + TailscaleNamespace: opts.tailscaleNamespace, + Logger: opts.log, + Clock: tstime.DefaultClock{}, + }), + proxyclass.NewReconciler(proxyclass.ReconcilerOptions{ + Client: mgr.GetClient(), + Recorder: eventRecorder, + TsNamespace: opts.tailscaleNamespace, + Logger: opts.log, + Clock: tstime.DefaultClock{}, + }), + dnsrecords.NewReconciler(dnsrecords.ReconcilerOptions{ + Client: mgr.GetClient(), + TailscaleNamespace: opts.tailscaleNamespace, + Logger: opts.log, + IsDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer, + }), + recorder.NewReconciler(recorder.ReconcilerOptions{ + Client: mgr.GetClient(), + Recorder: eventRecorder, + TailscaleNamespace: opts.tailscaleNamespace, + Clients: clients, + Logger: opts.log, + Clock: tstime.DefaultClock{}, + }), + egress.NewReconciler(egressOptions), + egress.NewReadinessReconciler(egressOptions), + egress.NewEndpointSliceReconciler(egressOptions), + egress.NewPodReconciler(egressOptions), } - proxyGroupPolicyOptions := proxygrouppolicy.ReconcilerOptions{ - Client: mgr.GetClient(), - } - - if err = proxygrouppolicy.NewReconciler(proxyGroupPolicyOptions).Register(mgr); err != nil { - startlog.Fatalf("could not register proxygrouppolicy reconciler: %v", err) - } - - peerRelayOptions := peerrelay.ReconcilerOptions{ - Client: mgr.GetClient(), - TailscaleNamespace: opts.tailscaleNamespace, - ProxyImage: opts.proxyImage, - DefaultTags: strings.Split(opts.proxyTags, ","), - Clients: clients, - Logger: opts.log, - } - - if err = peerrelay.NewReconciler(peerRelayOptions).Register(mgr); err != nil { - startlog.Fatalf("could not register peerrelay reconciler: %v", err) + for _, r := range reconcilers { + if err = r.Register(mgr); err != nil { + startlog.Fatalf("could not register %T: %v", r, err) + } } svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler) @@ -400,7 +443,6 @@ func runReconcilers(opts reconcilerOpts) { opts.proxyActAsDefaultLoadBalancer, )) - eventRecorder := mgr.GetEventRecorderFor("tailscale-operator") ssr := &tailscaleSTSReconciler{ Client: mgr.GetClient(), tsnetServer: opts.tsServer, @@ -557,131 +599,6 @@ func runReconcilers(opts reconcilerOpts) { if err != nil { startlog.Fatalf("could not create connector reconciler: %v", err) } - // TODO (irbekrm): switch to metadata-only watches for resources whose - // spec we don't need to inspect to reduce memory consumption. - // https://github.com/kubernetes-sigs/controller-runtime/issues/1159 - nameserverOptions := nameserver.ReconcilerOptions{ - Client: mgr.GetClient(), - Recorder: eventRecorder, - TailscaleNamespace: opts.tailscaleNamespace, - Logger: opts.log, - Clock: tstime.DefaultClock{}, - } - if err = nameserver.NewReconciler(nameserverOptions).Register(mgr); err != nil { - startlog.Fatalf("could not create nameserver reconciler: %v", err) - } - - egressSvcFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsHandler) - egressProxyGroupFilter := handler.EnqueueRequestsFromMapFunc(egressSvcsFromEgressProxyGroup(mgr.GetClient(), opts.log)) - err = builder. - ControllerManagedBy(mgr). - Named("egress-svcs-reconciler"). - Watches(&corev1.Service{}, egressSvcFilter). - Watches(&tsapi.ProxyGroup{}, egressProxyGroupFilter). - Complete(&egressSvcsReconciler{ - Client: mgr.GetClient(), - tsNamespace: opts.tailscaleNamespace, - recorder: eventRecorder, - clock: tstime.DefaultClock{}, - logger: opts.log.Named("egress-svcs-reconciler"), - }) - if err != nil { - startlog.Fatalf("could not create egress Services reconciler: %v", err) - } - if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexEgressProxyGroup, indexEgressServices); err != nil { - startlog.Fatalf("failed setting up indexer for egress Services: %v", err) - } - - egressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(egressSvcFromEps) - err = builder. - ControllerManagedBy(mgr). - Named("egress-svcs-readiness-reconciler"). - Watches(&corev1.Service{}, egressSvcFilter). - Watches(&discoveryv1.EndpointSlice{}, egressSvcFromEpsFilter). - Complete(&egressSvcsReadinessReconciler{ - Client: mgr.GetClient(), - tsNamespace: opts.tailscaleNamespace, - clock: tstime.DefaultClock{}, - logger: opts.log.Named("egress-svcs-readiness-reconciler"), - }) - if err != nil { - startlog.Fatalf("could not create egress Services readiness reconciler: %v", err) - } - - epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler) - podsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGPods(mgr.GetClient(), opts.tailscaleNamespace)) - secretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGStateSecrets(mgr.GetClient(), opts.tailscaleNamespace)) - epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace)) - - err = builder. - ControllerManagedBy(mgr). - Named("egress-eps-reconciler"). - Watches(&discoveryv1.EndpointSlice{}, epsFilter). - Watches(&corev1.Pod{}, podsFilter). - Watches(&corev1.Secret{}, secretsFilter). - Watches(&corev1.Service{}, epsFromExtNSvcFilter). - Complete(&egressEpsReconciler{ - Client: mgr.GetClient(), - tsNamespace: opts.tailscaleNamespace, - logger: opts.log.Named("egress-eps-reconciler"), - }) - if err != nil { - startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err) - } - - podsForEps := handler.EnqueueRequestsFromMapFunc(podsFromEgressEps(mgr.GetClient(), opts.log, opts.tailscaleNamespace)) - podsER := handler.EnqueueRequestsFromMapFunc(egressPodsHandler) - err = builder. - ControllerManagedBy(mgr). - Named("egress-pods-readiness-reconciler"). - Watches(&discoveryv1.EndpointSlice{}, podsForEps). - Watches(&corev1.Pod{}, podsER). - Complete(&egressPodsReconciler{ - Client: mgr.GetClient(), - tsNamespace: opts.tailscaleNamespace, - clock: tstime.DefaultClock{}, - logger: opts.log.Named("egress-pods-readiness-reconciler"), - httpClient: http.DefaultClient, - }) - if err != nil { - startlog.Fatalf("could not create egress Pods readiness reconciler: %v", err) - } - - proxyClassOptions := proxyclass.ReconcilerOptions{ - Client: mgr.GetClient(), - Recorder: eventRecorder, - TsNamespace: opts.tailscaleNamespace, - Logger: opts.log, - Clock: tstime.DefaultClock{}, - } - - if err = proxyclass.NewReconciler(proxyClassOptions).Register(mgr); err != nil { - startlog.Fatalf("could not create proxyclass reconciler: %v", err) - } - - dnsRecordsOptions := dnsrecords.ReconcilerOptions{ - Client: mgr.GetClient(), - TailscaleNamespace: opts.tailscaleNamespace, - Logger: opts.log, - IsDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer, - } - if err = dnsrecords.NewReconciler(dnsRecordsOptions).Register(mgr); err != nil { - startlog.Fatalf("could not create DNS records reconciler: %v", err) - } - - recorderOptions := recorder.ReconcilerOptions{ - Client: mgr.GetClient(), - Recorder: eventRecorder, - TailscaleNamespace: opts.tailscaleNamespace, - Clients: clients, - Logger: opts.log, - Clock: tstime.DefaultClock{}, - } - - if err = recorder.NewReconciler(recorderOptions).Register(mgr); err != nil { - startlog.Fatalf("could not create Recorder reconciler: %v", err) - } - // kube-apiserver's Tailscale Service reconciler. err = builder. ControllerManagedBy(mgr). @@ -1206,101 +1123,6 @@ func serviceHandler(_ context.Context, o client.Object) []reconcile.Request { } } -// isMagicDNSName reports whether name is a full tailnet node FQDN (with or -// without final dot). -func isMagicDNSName(name string) bool { - validMagicDNSName := regexp.MustCompile(`^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+\.ts\.net\.?$`) - return validMagicDNSName.MatchString(name) -} - -// egressSvcsHandler returns accepts a Kubernetes object and returns a reconcile -// request for it , if the object is a Tailscale egress Service meant to be -// exposed on a ProxyGroup. -func egressSvcsHandler(_ context.Context, o client.Object) []reconcile.Request { - if !isEgressSvcForProxyGroup(o) { - return nil - } - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: o.GetNamespace(), - Name: o.GetName(), - }, - }, - } -} - -// egressEpsHandler returns accepts an EndpointSlice and, if the EndpointSlice -// is for an egress service, returns a reconcile request for it. -func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request { - if typ := o.GetLabels()[labelSvcType]; typ != typeEgress { - return nil - } - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: o.GetNamespace(), - Name: o.GetName(), - }, - }, - } -} - -func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request { - if typ := o.GetLabels()[LabelParentType]; typ != proxyTypeProxyGroup { - return nil - } - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: o.GetNamespace(), - Name: o.GetName(), - }, - }, - } -} - -// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is, -// returns reconciler requests for all egress EndpointSlices for that ProxyGroup. -func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc { - return func(_ context.Context, o client.Object) []reconcile.Request { - if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { - return nil - } - // TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we - // have ingress ProxyGroups. - if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" { - return nil - } - pg, ok := o.GetLabels()[LabelParentName] - if !ok { - return nil - } - return reconcileRequestsForPG(pg, cl, ns) - } -} - -// egressEpsFromPGStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is, -// returns reconciler requests for all egress EndpointSlices for that ProxyGroup. -func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc { - return func(_ context.Context, o client.Object) []reconcile.Request { - if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { - return nil - } - if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" { - return nil - } - if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState { - return nil - } - pg, ok := o.GetLabels()[LabelParentName] - if !ok { - return nil - } - return reconcileRequestsForPG(pg, cl, ns) - } -} - func ingressSvcFromEps(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { svcName := o.GetLabels()[discoveryv1.LabelServiceName] @@ -1333,52 +1155,6 @@ func ingressSvcFromEps(cl client.Client, logger *zap.SugaredLogger) handler.MapF } } -// egressSvcFromEps is an event handler for EndpointSlices. If an EndpointSlice is for an egress ExternalName Service -// meant to be exposed on a ProxyGroup, returns a reconcile request for the Service. -func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request { - if typ := o.GetLabels()[labelSvcType]; typ != typeEgress { - return nil - } - if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { - return nil - } - svcName, ok := o.GetLabels()[LabelParentName] - if !ok { - return nil - } - svcNs, ok := o.GetLabels()[LabelParentNamespace] - if !ok { - return nil - } - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: svcNs, - Name: svcName, - }, - }, - } -} - -func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.Request { - epsList := discoveryv1.EndpointSliceList{} - if err := cl.List(context.Background(), &epsList, - client.InNamespace(ns), - client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil { - return nil - } - reqs := make([]reconcile.Request, 0) - for _, ep := range epsList.Items { - reqs = append(reqs, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: ep.Namespace, - Name: ep.Name, - }, - }) - } - return reqs -} - func isTLSSecret(secret *corev1.Secret) bool { return secret.Type == corev1.SecretTypeTLS && secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" && @@ -1513,37 +1289,6 @@ func kubeAPIServerPGsFromSecret(cl client.Client, logger *zap.SugaredLogger) han } } -// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all -// user-created ExternalName Services that should be exposed on this ProxyGroup. -func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { - return func(ctx context.Context, o client.Object) []reconcile.Request { - pg, ok := o.(*tsapi.ProxyGroup) - if !ok { - logger.Warn("ProxyGroup handler triggered for an object that is not a ProxyGroup") - return nil - } - - if pg.Spec.Type != tsapi.ProxyGroupTypeEgress { - return nil - } - svcList := &corev1.ServiceList{} - if err := cl.List(ctx, svcList, client.MatchingFields{indexEgressProxyGroup: pg.Name}); err != nil { - logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name) - return nil - } - reqs := make([]reconcile.Request, 0) - for _, svc := range svcList.Items { - reqs = append(reqs, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: svc.Namespace, - Name: svc.Name, - }, - }) - } - return reqs - } -} - // ingressesFromIngressProxyGroup is an event handler for ingress ProxyGroups. It returns reconcile requests for all // user-created Ingresses that should be exposed on this ProxyGroup. func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { @@ -1575,76 +1320,6 @@ func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger) } } -// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that -// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service. -func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { - return func(ctx context.Context, o client.Object) []reconcile.Request { - svc, ok := o.(*corev1.Service) - if !ok { - logger.Warn("Service handler triggered for an object that is not a Service") - return nil - } - - if !isEgressSvcForProxyGroup(svc) { - return nil - } - epsList := &discoveryv1.EndpointSliceList{} - if err := cl.List(ctx, epsList, client.InNamespace(ns), - client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil { - logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name) - return nil - } - reqs := make([]reconcile.Request, 0) - for _, eps := range epsList.Items { - reqs = append(reqs, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: eps.Namespace, - Name: eps.Name, - }, - }) - } - return reqs - } -} - -func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { - return func(ctx context.Context, o client.Object) []reconcile.Request { - eps, ok := o.(*discoveryv1.EndpointSlice) - if !ok { - logger.Warn("EndpointSlice handler triggered for an object that is not a EndpointSlice") - return nil - } - - if eps.Labels[labelProxyGroup] == "" { - return nil - } - if eps.Labels[labelSvcType] != "egress" { - return nil - } - podLabels := map[string]string{ - kubetypes.LabelManaged: "true", - LabelParentType: "proxygroup", - LabelParentName: eps.Labels[labelProxyGroup], - } - podList := &corev1.PodList{} - if err := cl.List(ctx, podList, client.InNamespace(ns), - client.MatchingLabels(podLabels)); err != nil { - logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on EndpointSlice %s", err, eps.Name) - return nil - } - reqs := make([]reconcile.Request, 0) - for _, pod := range podList.Items { - reqs = append(reqs, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: pod.Namespace, - Name: pod.Name, - }, - }) - } - return reqs - } -} - // crdTransformer gets called before a CRD is stored to c/r cache, it removes the CRD spec to reduce memory consumption. func crdTransformer(log *zap.SugaredLogger) toolscache.TransformFunc { return func(o any) (any, error) { @@ -1659,15 +1334,6 @@ func crdTransformer(log *zap.SugaredLogger) toolscache.TransformFunc { } } -// indexEgressServices adds a local index to cached Tailscale egress Services meant to be exposed on a ProxyGroup. The -// index is used a list filter. -func indexEgressServices(o client.Object) []string { - if !isEgressSvcForProxyGroup(o) { - return nil - } - return []string{o.GetAnnotations()[AnnotationProxyGroup]} -} - // indexPGIngresses is used to select ProxyGroup-backed Services which are // locally indexed in the cache for efficient listing without requiring labels. func indexPGIngresses(o client.Object) []string { diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index b775a36fb..1dc39cdb1 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -27,12 +27,11 @@ "tailscale.com/k8s-operator/apis/v1alpha1" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" - "tailscale.com/net/dns/resolvconffile" "tailscale.com/tstest" "tailscale.com/tstime" - "tailscale.com/util/dnsname" "tailscale.com/util/mak" ) @@ -1522,8 +1521,8 @@ func Test_isMagicDNSName(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := isMagicDNSName(tt.in); got != tt.want { - t.Errorf("isMagicDNSName(%q) = %v, want %v", tt.in, got, tt.want) + if got := reconciler.IsMagicDNSName(tt.in); got != tt.want { + t.Errorf("reconciler.IsMagicDNSName(%q) = %v, want %v", tt.in, got, tt.want) } }) } @@ -1753,72 +1752,6 @@ func Test_serviceHandlerForIngress_multipleIngressClasses(t *testing.T) { } } -func Test_clusterDomainFromResolverConf(t *testing.T) { - zl := zap.Must(zap.NewDevelopment()) - tests := []struct { - name string - conf *resolvconffile.Config - namespace string - want string - }{ - { - name: "success-custom-domain", - conf: &resolvconffile.Config{ - SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")}, - }, - namespace: "foo", - want: "department.org.io", - }, - { - name: "success-default-domain", - conf: &resolvconffile.Config{ - SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.cluster.local."), toFQDN(t, "svc.cluster.local."), toFQDN(t, "cluster.local.")}, - }, - namespace: "foo", - want: "cluster.local", - }, - { - name: "only-two-search-domains", - conf: &resolvconffile.Config{ - SearchDomains: []dnsname.FQDN{toFQDN(t, "svc.department.org.io"), toFQDN(t, "department.org.io")}, - }, - namespace: "foo", - want: "cluster.local", - }, - { - name: "first-search-domain-mismatch", - conf: &resolvconffile.Config{ - SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.bar.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")}, - }, - namespace: "foo", - want: "cluster.local", - }, - { - name: "second-search-domain-mismatch", - conf: &resolvconffile.Config{ - SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "foo.department.org.io"), toFQDN(t, "some.other.fqdn")}, - }, - namespace: "foo", - want: "cluster.local", - }, - { - name: "third-search-domain-mismatch", - conf: &resolvconffile.Config{ - SearchDomains: []dnsname.FQDN{toFQDN(t, "foo.svc.department.org.io"), toFQDN(t, "svc.department.org.io"), toFQDN(t, "some.other.fqdn")}, - }, - namespace: "foo", - want: "cluster.local", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := clusterDomainFromResolverConf(tt.conf, tt.namespace, zl.Sugar()); got != tt.want { - t.Errorf("clusterDomainFromResolverConf() = %v, want %v", got, tt.want) - } - }) - } -} - func Test_authKeyRemoval(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} @@ -2095,15 +2028,6 @@ func TestIgnorePGService(t *testing.T) { findNoGenName(t, fc, "default", "test", "svc") } -func toFQDN(t *testing.T, s string) dnsname.FQDN { - t.Helper() - fqdn, err := dnsname.ToFQDN(s) - if err != nil { - t.Fatalf("error coverting %q to dnsname.FQDN: %v", s, err) - } - return fqdn -} - func proxyCreatedCondition(clock tstime.Clock) []metav1.Condition { return []metav1.Condition{{ Type: string(tsapi.ProxyReady), diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 05ffae293..59936500f 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -34,6 +34,7 @@ "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/egressservices" @@ -205,7 +206,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, tsClient tsclien if err != nil { return r.notReadyErrf(pg, logger, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err) } - if !tsoperator.ProxyClassIsReady(proxyClass) { + if !reconciler.ProxyClassIsReady(proxyClass) { msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName) logger.Info(msg) return notReady(reasonProxyGroupCreating, msg) @@ -487,7 +488,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za pg.Status.Devices = devices - desiredReplicas := int(pgReplicas(pg)) + desiredReplicas := int(reconciler.ProxyGroupReplicas(pg)) // Set ProxyGroupAvailable condition. status := metav1.ConditionFalse @@ -499,10 +500,10 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za reason = reasonProxyGroupAvailable } } - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger) // Set ProxyGroupReady condition. - tsSvcValid, tsSvcSet := tsoperator.KubeAPIServerProxyValid(pg) + tsSvcValid, tsSvcSet := reconciler.KubeAPIServerProxyValid(pg) status = metav1.ConditionFalse reason = reasonProxyGroupCreating switch { @@ -516,7 +517,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za case len(devices) < desiredReplicas: case len(devices) > desiredReplicas: message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(devices)-desiredReplicas) - case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !tsoperator.KubeAPIServerProxyConfigured(pg): + case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !reconciler.KubeAPIServerProxyConfigured(pg): reason = reasonProxyGroupCreating message = "waiting for proxies to start advertising the kube-apiserver proxy's hostname" default: @@ -524,7 +525,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za reason = reasonProxyGroupReady message = reasonProxyGroupReady } - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger) return nil } @@ -566,14 +567,14 @@ func (e *allocatePortsErr) Error() string { } func (r *ProxyGroupReconciler) allocatePorts(ctx context.Context, pg *tsapi.ProxyGroup, proxyClassName string, portRanges tsapi.PortRanges) (map[string]uint16, error) { - replicaCount := int(pgReplicas(pg)) + replicaCount := int(reconciler.ProxyGroupReplicas(pg)) svcToNodePorts, usedPorts, err := getServicePortsForProxyGroups(ctx, r.Client, r.tsNamespace, portRanges) if err != nil { return nil, &allocatePortsErr{msg: fmt.Sprintf("failed to find ports for existing ProxyGroup NodePort Services: %s", err.Error())} } replicasAllocated := 0 - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { if _, ok := svcToNodePorts[pgNodePortServiceName(pg.Name, i)]; !ok { svcToNodePorts[pgNodePortServiceName(pg.Name, i)] = 0 } else { @@ -605,7 +606,7 @@ func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, // NOTE: (ChaosInTheCRD) we want the same TargetPort for every static endpoint NodePort Service for the ProxyGroup tailscaledPort := getRandomPort() svcs := []*corev1.Service{} - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { nodePortSvcName := pgNodePortServiceName(pg.Name, i) svc := &corev1.Service{} @@ -663,7 +664,7 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, tsC } for _, m := range metadata { - if m.ordinal+1 <= pgReplicas(pg) { + if m.ordinal+1 <= reconciler.ProxyGroupReplicas(pg) { continue } @@ -750,8 +751,8 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated( svcToNodePorts map[string]uint16, ) (endpoints map[string][]netip.AddrPort, err error) { logger := r.logger(pg.Name) - endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) // keyed by Service name. - for i := range pgReplicas(pg) { + endpoints = make(map[string][]netip.AddrPort, reconciler.ProxyGroupReplicas(pg)) // keyed by Service name. + for i := range reconciler.ProxyGroupReplicas(pg) { logger = logger.With("Pod", fmt.Sprintf("%s-%d", pg.Name, i)) cfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -1116,7 +1117,7 @@ func (r *ProxyGroupReconciler) ensureStateAddedForProxyGroup(pg *tsapi.ProxyGrou gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) - r.reissuer.EnsureState(pg.Name, int(pgReplicas(pg))) + r.reissuer.EnsureState(pg.Name, int(reconciler.ProxyGroupReplicas(pg))) } // ensureStateRemovedForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 6337fec55..0ff7bd41b 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -21,6 +21,8 @@ "sigs.k8s.io/yaml" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/egress" "tailscale.com/kube/egressservices" "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubetypes" @@ -92,7 +94,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Labels: pgLabels(pg.Name, nil), OwnerReferences: pgOwnerReference(pg), } - ss.Spec.Replicas = new(pgReplicas(pg)) + ss.Spec.Replicas = new(reconciler.ProxyGroupReplicas(pg)) ss.Spec.Selector = &metav1.LabelSelector{ MatchLabels: pgLabels(pg.Name, nil), } @@ -107,13 +109,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string } tmpl.Spec.ServiceAccountName = pg.Name tmpl.Spec.InitContainers[0].Image = image - proxyConfigVolName := pgEgressCMName(pg.Name) + proxyConfigVolName := egress.CMName(pg.Name) if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { proxyConfigVolName = pgIngressCMName(pg.Name) } tmpl.Spec.Volumes = func() []corev1.Volume { var volumes []corev1.Volume - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { volumes = append(volumes, corev1.Volume{ Name: fmt.Sprintf("tailscaledconfig-%d", i), VolumeSource: corev1.VolumeSource{ @@ -147,7 +149,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string // TODO(tomhjp): Read config directly from the secret instead. The // mounts change on scaling up/down which causes unnecessary restarts // for pods that haven't meaningfully changed. - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { mounts = append(mounts, corev1.VolumeMount{ Name: fmt.Sprintf("tailscaledconfig-%d", i), ReadOnly: true, @@ -313,7 +315,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string // cluster egress traffic. The reconciler sets the corresponding // condition; see egress-pod-readiness.go. tmpl.Spec.ReadinessGates = append(tmpl.Spec.ReadinessGates, corev1.PodReadinessGate{ - ConditionType: tsEgressReadinessGate, + ConditionType: egress.ReadinessGate, }) } @@ -329,7 +331,7 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por OwnerReferences: pgOwnerReference(pg), }, Spec: appsv1.StatefulSetSpec{ - Replicas: new(pgReplicas(pg)), + Replicas: new(reconciler.ProxyGroupReplicas(pg)), Selector: &metav1.LabelSelector{ MatchLabels: pgLabels(pg.Name, nil), }, @@ -464,7 +466,7 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string, shareACMEAccount bool) *rbac "update", }, ResourceNames: func() (secrets []string) { - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { secrets = append(secrets, pgConfigSecretName(pg.Name, i), // Config with auth key. pgPodName(pg.Name, i), // State. @@ -565,7 +567,7 @@ func isAuthAPIServerProxy(pg *tsapi.ProxyGroup) bool { } func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) { - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { secrets = append(secrets, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: pgStateSecretName(pg.Name, i), @@ -584,7 +586,7 @@ func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) (*corev1.ConfigMap, []by hpBs := []byte(strconv.Itoa(hp)) return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: pgEgressCMName(pg.Name), + Name: egress.CMName(pg.Name), Namespace: namespace, Labels: pgLabels(pg.Name, nil), OwnerReferences: pgOwnerReference(pg), @@ -625,14 +627,6 @@ func pgOwnerReference(owner *tsapi.ProxyGroup) []metav1.OwnerReference { return []metav1.OwnerReference{*metav1.NewControllerRef(owner, tsapi.SchemeGroupVersion.WithKind("ProxyGroup"))} } -func pgReplicas(pg *tsapi.ProxyGroup) int32 { - if pg.Spec.Replicas != nil { - return *pg.Spec.Replicas - } - - return 2 -} - func pgPodName(pgName string, i int32) string { return fmt.Sprintf("%s-%d", pgName, i) } @@ -653,10 +647,6 @@ func pgStateSecretName(pgName string, i int32) string { return fmt.Sprintf("%s-%d", pgName, i) } -func pgEgressCMName(pg string) string { - return fmt.Sprintf("%s-egress-config", pg) -} - // hasLocalAddrPortSet returns true if the proxyclass has the TS_LOCAL_ADDR_PORT env var set. For egress ProxyGroups, // currently (2025-01-26) this means that the ProxyGroup does not support graceful failover. func hasLocalAddrPortSet(proxyClass *tsapi.ProxyClass) bool { @@ -671,7 +661,7 @@ func hasLocalAddrPortSet(proxyClass *tsapi.ProxyClass) bool { // hepPings returns the number of times a health check endpoint exposed by a Service fronting ProxyGroup replicas should // be pinged to ensure that all currently configured backend replicas are hit. func hepPings(pg *tsapi.ProxyGroup) int { - rc := pgReplicas(pg) + rc := reconciler.ProxyGroupReplicas(pg) // Assuming a Service implemented using round robin load balancing, number-of-replica-times should be enough, but in // practice, we cannot assume that the requests will be load balanced perfectly. return int(rc) * 3 diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 4e992aafb..954d0a5ea 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -32,6 +32,8 @@ "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/egress" "tailscale.com/k8s-operator/reconciler/proxyclass" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" @@ -633,7 +635,7 @@ type reconcile struct { WithScheme(tsapi.GlobalScheme). Build() - reconciler := &ProxyGroupReconciler{ + pgr := &ProxyGroupReconciler{ tsNamespace: tsNamespace, tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, @@ -673,7 +675,7 @@ type reconcile struct { t.Logf("created node %q with data", n.name) } - reconciler.log = zl.Sugar().With("TestName", tt.name).With("Reconcile", i) + pgr.log = zl.Sugar().With("TestName", tt.name).With("Reconcile", i) pg.Spec.Replicas = r.replicas pc.Spec.StaticEndpoints = r.staticEndpointConfig @@ -686,9 +688,9 @@ type reconcile struct { }) if r.expectedErr != "" { - expectError(t, reconciler, "", pg.Name) + expectError(t, pgr, "", pg.Name) } else { - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) } expectEvents(t, fr, r.expectedEvents) @@ -777,7 +779,7 @@ type reconcile struct { } t.Run("delete_and_cleanup", func(t *testing.T) { - reconciler := &ProxyGroupReconciler{ + pgr := &ProxyGroupReconciler{ tsNamespace: tsNamespace, tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, @@ -796,7 +798,7 @@ type reconcile struct { t.Fatalf("error deleting ProxyGroup: %v", err) } - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name) if err := fc.Delete(t.Context(), pc); err != nil { @@ -923,7 +925,7 @@ func TestProxyGroup(t *testing.T) { zl, _ := zap.NewDevelopment() fr := record.NewFakeRecorder(1) cl := tstest.NewClock(tstest.ClockOpts{}) - reconciler := &ProxyGroupReconciler{ + pgr := &ProxyGroupReconciler{ tsNamespace: tsNamespace, tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, @@ -948,13 +950,13 @@ func TestProxyGroup(t *testing.T) { } t.Run("proxyclass_not_ready", func(t *testing.T) { - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, false, pc) - if tsoperator.ProxyGroupAvailable(pg) { + if reconciler.ProxyGroupAvailable(pg) { t.Fatal("expected ProxyGroup to not be available") } }) @@ -976,17 +978,17 @@ func TestProxyGroup(t *testing.T) { mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { p.ObjectMeta.Generation = pg.ObjectMeta.Generation }) - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 2, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 2, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) - if tsoperator.ProxyGroupAvailable(pg) { + if reconciler.ProxyGroupAvailable(pg) { t.Fatal("expected ProxyGroup to not be available") } - if expected := 1; reconciler.egressProxyGroups.Len() != expected { - t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) + if expected := 1; pgr.egressProxyGroups.Len() != expected { + t.Fatalf("expected %d egress ProxyGroups, got %d", expected, pgr.egressProxyGroups.Len()) } expectProxyGroupResources(t, fc, pg, true, pc) var keyReq tailscale.KeyCapabilities @@ -1006,7 +1008,7 @@ func TestProxyGroup(t *testing.T) { mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { p.ObjectMeta.Generation = pg.ObjectMeta.Generation }) - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) pg.Status.Devices = []tsapi.TailnetDevice{ { @@ -1018,11 +1020,11 @@ func TestProxyGroup(t *testing.T) { TailnetIPs: []string{"1.2.3.4", "::1"}, }, } - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) - if !tsoperator.ProxyGroupAvailable(pg) { + if !reconciler.ProxyGroupAvailable(pg) { t.Fatal("expected ProxyGroup to be available") } }) @@ -1032,16 +1034,16 @@ func TestProxyGroup(t *testing.T) { mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { p.Spec = pg.Spec }) - expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 3, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + expectReconciled(t, pgr, "", pg.Name) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 3, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) addNodeIDToStateSecrets(t, fc, pg) - expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + expectReconciled(t, pgr, "", pg.Name) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ Hostname: "hostname-nodeid-2", TailnetIPs: []string{"1.2.3.4", "::1"}, @@ -1056,10 +1058,10 @@ func TestProxyGroup(t *testing.T) { p.Spec = pg.Spec }) - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) }) @@ -1069,7 +1071,7 @@ func TestProxyGroup(t *testing.T) { mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { p.Spec = pc.Spec }) - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) expectEqual(t, fc, expectedMetricsService(opts)) }) t.Run("enable_service_monitor_no_crd", func(t *testing.T) { @@ -1077,11 +1079,11 @@ func TestProxyGroup(t *testing.T) { mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { p.Spec.Metrics = pc.Spec.Metrics }) - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) }) t.Run("create_crd_expect_service_monitor", func(t *testing.T) { mustCreate(t, fc, crd) - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts)) }) @@ -1090,18 +1092,18 @@ func TestProxyGroup(t *testing.T) { t.Fatal(err) } - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name) - if expected := 0; reconciler.egressProxyGroups.Len() != expected { - t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) + if expected := 0; pgr.egressProxyGroups.Len() != expected { + t.Fatalf("expected %d ProxyGroups, got %d", expected, pgr.egressProxyGroups.Len()) } // 2 nodes should get deleted as part of the scale down, and then finally // the first node gets deleted with the ProxyGroup cleanup. if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-1", "nodeid-2", "nodeid-0"}); diff != "" { t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) } - expectMissing[corev1.Service](t, reconciler, "tailscale", metricsResourceName(pg.Name)) + expectMissing[corev1.Service](t, pgr, "tailscale", metricsResourceName(pg.Name)) // The fake client does not clean up objects whose owner has been // deleted, so we can't test for the owned resources getting deleted. }) @@ -1131,7 +1133,7 @@ func TestProxyGroupTypes(t *testing.T) { }) zl, _ := zap.NewDevelopment() - reconciler := &ProxyGroupReconciler{ + pgr := &ProxyGroupReconciler{ tsNamespace: tsNamespace, tsProxyImage: testProxyImage, Client: fc, @@ -1155,8 +1157,8 @@ func TestProxyGroupTypes(t *testing.T) { } mustCreate(t, fc, pg) - expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 0, 1, 0) + expectReconciled(t, pgr, "", pg.Name) + verifyProxyGroupCounts(t, pgr, 0, 1, 0) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1217,9 +1219,9 @@ func TestProxyGroupTypes(t *testing.T) { t.Errorf("unexpected deletion grace period seconds %d, want %d", *sts.Spec.Template.DeletionGracePeriodSeconds, deletionGracePeriodSeconds) } if !slices.ContainsFunc(sts.Spec.Template.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool { - return r.ConditionType == tsEgressReadinessGate + return r.ConditionType == egress.ReadinessGate }) { - t.Errorf("expected egress readiness gate %q to be set, got %v", tsEgressReadinessGate, sts.Spec.Template.Spec.ReadinessGates) + t.Errorf("expected egress readiness gate %q to be set, got %v", egress.ReadinessGate, sts.Spec.Template.Spec.ReadinessGates) } }) t.Run("egress_type_no_lifecycle_hook_when_local_addr_port_set", func(t *testing.T) { @@ -1247,7 +1249,7 @@ func TestProxyGroupTypes(t *testing.T) { }, } }) - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1258,7 +1260,7 @@ func TestProxyGroupTypes(t *testing.T) { t.Error("lifecycle hook was set when TS_LOCAL_ADDR_PORT was configured via ProxyClass") } if slices.ContainsFunc(sts.Spec.Template.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool { - return r.ConditionType == tsEgressReadinessGate + return r.ConditionType == egress.ReadinessGate }) { t.Error("egress readiness gate was set when TS_LOCAL_ADDR_PORT was configured via ProxyClass") } @@ -1282,8 +1284,8 @@ func TestProxyGroupTypes(t *testing.T) { t.Fatal(err) } - expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 1, 2, 0) + expectReconciled(t, pgr, "", pg.Name) + verifyProxyGroupCounts(t, pgr, 1, 2, 0) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1381,7 +1383,7 @@ func TestProxyGroupTypes(t *testing.T) { if err := fc.Create(t.Context(), pg); err != nil { t.Fatal(err) } - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1431,8 +1433,8 @@ func TestProxyGroupTypes(t *testing.T) { t.Fatal(err) } - expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 2, 2, 1) + expectReconciled(t, pgr, "", pg.Name) + verifyProxyGroupCounts(t, pgr, 2, 2, 1) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1490,17 +1492,17 @@ func TestKubeAPIServerStatusConditionFlow(t *testing.T) { expectReconciled(t, r, "", pg.Name) pg.ObjectMeta.Finalizers = append(pg.ObjectMeta.Finalizers, FinalizerName) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.log) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.log) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Set kube-apiserver valid. mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { - tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log) + reconciler.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log) }) expectReconciled(t, r, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.log) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Set available. @@ -1512,17 +1514,17 @@ func TestKubeAPIServerStatusConditionFlow(t *testing.T) { TailnetIPs: []string{"1.2.3.4", "::1"}, }, } - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.log) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.log) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.log) expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Set kube-apiserver configured. mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { - tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log) + reconciler.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log) }) expectReconciled(t, r, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.log) + reconciler.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.log) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.log) expectEqual(t, fc, pg, omitPGStatusConditionMessages) } @@ -1532,7 +1534,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { WithStatusSubresource(&tsapi.ProxyGroup{}). Build() - reconciler := &ProxyGroupReconciler{ + pgr := &ProxyGroupReconciler{ tsNamespace: tsNamespace, tsProxyImage: testProxyImage, Client: fc, @@ -1558,7 +1560,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { if err := fc.Create(t.Context(), pg); err != nil { t.Fatal(err) } - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) cfg := conf.VersionedConfig{ Version: "v1alpha1", @@ -1607,7 +1609,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { mustUpdate(t, fc, tsNamespace, cfgSecret.Name, func(s *corev1.Secret) { s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB }) - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) cfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cfgB expectEqual(t, fc, cfgSecret) @@ -1618,7 +1620,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { WithScheme(tsapi.GlobalScheme). WithStatusSubresource(&tsapi.ProxyGroup{}). Build() - reconciler := &ProxyGroupReconciler{ + pgr := &ProxyGroupReconciler{ tsNamespace: tsNamespace, tsProxyImage: testProxyImage, Client: fc, @@ -1658,7 +1660,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { Replicas: new(int32(1)), }, }) - expectReconciled(t, reconciler, "", pgName) + expectReconciled(t, pgr, "", pgName) expectedConfigBytes, err := json.Marshal(ipn.ConfigVAlpha{ // Preserved. @@ -1885,7 +1887,7 @@ func TestProxyGroupGetAuthKey(t *testing.T) { zl, _ := zap.NewDevelopment() fr := record.NewFakeRecorder(1) cl := tstest.NewClock(tstest.ClockOpts{}) - reconciler := &ProxyGroupReconciler{ + pgr := &ProxyGroupReconciler{ tsNamespace: tsNamespace, tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, @@ -1898,9 +1900,9 @@ func TestProxyGroupGetAuthKey(t *testing.T) { clock: cl, reissuer: tailscaled.NewReissuer(), } - reconciler.ensureStateAddedForProxyGroup(pg) + pgr.ensureStateAddedForProxyGroup(pg) - return reconciler, fc + return pgr, fc } // Config Secret: exists or not, has key or not. @@ -1970,7 +1972,7 @@ func TestProxyGroupGetAuthKey(t *testing.T) { } { t.Run(name, func(t *testing.T) { tsClient.deleted = tsClient.deleted[:0] // Reset deleted devices for each test case. - reconciler, fc := initTest() + pgr, fc := initTest() var cfgSecret *corev1.Secret if tc.configData != nil { cfgSecret = &corev1.Secret{ @@ -1991,7 +1993,7 @@ func TestProxyGroupGetAuthKey(t *testing.T) { }) } - authKey, err := reconciler.getAuthKey(t.Context(), tsClient, pg, cfgSecret, 0, reconciler.log.With("TestName", t.Name())) + authKey, err := pgr.getAuthKey(t.Context(), tsClient, pg, cfgSecret, 0, pgr.log.With("TestName", t.Name())) if err != nil { t.Fatalf("unexpected error getting auth key: %v", err) } @@ -2135,7 +2137,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox var expectedSecrets []string if shouldExist { - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { expectedSecrets = append(expectedSecrets, fmt.Sprintf("%s-%d", pg.Name, i), pgConfigSecretName(pg.Name, i), @@ -2166,7 +2168,7 @@ func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) { func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) { t.Helper() const key = "profile-abc" - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { bytes, err := json.Marshal(map[string]any{ "Config": map[string]any{ "NodeID": fmt.Sprintf("nodeid-%d", i), @@ -2260,7 +2262,7 @@ type proxyGroupLETestCase struct { setProxyClassReady(t, fc, cl, name) } - reconciler := &ProxyGroupReconciler{ + pgr := &ProxyGroupReconciler{ tsNamespace: tsNamespace, tsProxyImage: testProxyImage, defaultTags: []string{"tag:test"}, @@ -2272,7 +2274,7 @@ type proxyGroupLETestCase struct { reissuer: tailscaled.NewReissuer(), } - expectReconciled(t, reconciler, "", pg.Name) + expectReconciled(t, pgr, "", pg.Name) // Verify that the StatefulSet created for ProxyGrup has // the expected setting for the staging endpoint. diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index ebd9a5b1b..de925de22 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -34,6 +34,7 @@ "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/kubetypes" "tailscale.com/net/netutil" @@ -68,6 +69,11 @@ AnnotationProxyGroup = "tailscale.com/proxy-group" + // labelProxyGroup names the ProxyGroup a managed resource belongs to. Same string as + // AnnotationProxyGroup, which users set on the Service; this is the label the operator + // stamps on the resources it creates in response. + labelProxyGroup = "tailscale.com/proxy-group" + // AnnotationShareACMEAccount opts a single ProxyGroup into ("true") // or out of ("false") using the shared per-tailnet ACME account key. // When absent, OPERATOR_SHARED_ACME_ACCOUNT_KEY on the operator is @@ -216,7 +222,7 @@ func (r *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga if err := r.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil { return nil, fmt.Errorf("failed to get ProxyClass: %w", err) } - if !tsoperator.ProxyClassIsReady(proxyClass) { + if !reconciler.ProxyClassIsReady(proxyClass) { logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..") return nil, nil } @@ -1288,13 +1294,6 @@ func defaultEnv(envName, defVal string) string { return v } -func nameForService(svc *corev1.Service) string { - if h, ok := svc.Annotations[AnnotationHostname]; ok { - return h - } - return svc.Namespace + "-" + svc.Name -} - // markedForDeletion reports whether obj has a deletion timestamp, i.e. the API server is waiting on finalizers // before it can garbage collect it. func markedForDeletion(obj metav1.Object) bool { diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 07d5843ff..002e09afb 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -29,8 +29,8 @@ "tailscale.com/client/tailscale/v2" "tailscale.com/ipn" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/ingressservices" @@ -116,7 +116,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque return res, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) } - if !tsoperator.ProxyGroupAvailable(pg) { + if !reconciler.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") return res, nil } @@ -126,7 +126,7 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque return res, fmt.Errorf("failed to get tailscale client: %w", err) } - hostname := nameForService(svc) + hostname := reconciler.NameForService(svc) logger = logger.With("hostname", hostname) if !svc.DeletionTimestamp.IsZero() || !r.isTailscaleService(svc) { @@ -173,7 +173,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin if err = r.validateService(ctx, svc, pg); err != nil { r.recorder.Event(svc, corev1.EventTypeWarning, reasonIngressSvcInvalid, err.Error()) - tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, err.Error(), r.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, err.Error(), r.clock, logger) return false, nil } @@ -224,7 +224,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) logger.Warn(msg) r.recorder.Event(svc, corev1.EventTypeWarning, "InvalidTailscaleService", msg) - tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, msg, r.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, msg, r.clock, logger) return false, nil } @@ -347,7 +347,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin conditionStatus := metav1.ConditionFalse conditionType := tsapi.IngressSvcConfigured conditionReason := reasonIngressSvcNoBackendsConfigured - conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", count, pgReplicas(pg)) + conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", count, reconciler.ProxyGroupReplicas(pg)) if count != 0 { dnsName, err := dnsNameForService(ctx, r.Client, serviceName, pg, r.tsNamespace) if err != nil { @@ -365,7 +365,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin conditionReason = reasonIngressSvcConfigured } - tsoperator.SetServiceCondition(svc, conditionType, conditionStatus, conditionReason, conditionMessage, r.clock, logger) + reconciler.SetServiceCondition(svc, conditionType, conditionStatus, conditionReason, conditionMessage, r.clock, logger) svc.Status.LoadBalancer.Ingress = lbs return svcsChanged, nil @@ -441,7 +441,7 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG for tsSvcName, cfg := range config { found := false for _, svc := range svcList.Items { - if strings.EqualFold(fmt.Sprintf("svc:%s", nameForService(&svc)), tsSvcName) { + if strings.EqualFold(fmt.Sprintf("svc:%s", reconciler.NameForService(&svc)), tsSvcName) { found = true break } @@ -820,7 +820,7 @@ func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.S errs = append(errs, fmt.Errorf("ProxyGroup %q is of type %q but must be of type %q", pg.Name, pg.Spec.Type, tsapi.ProxyGroupTypeIngress)) } - if violations := validateService(svc); len(violations) > 0 { + if violations := reconciler.ValidateService(svc); len(violations) > 0 { errs = append(errs, fmt.Errorf("invalid Service: %s", strings.Join(violations, ", "))) } svcList := &corev1.ServiceList{} @@ -828,7 +828,7 @@ func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.S errs = append(errs, fmt.Errorf("error listing Services: %w", err)) return errors.Join(errs...) } - svcName := nameForService(svc) + svcName := reconciler.NameForService(svc) for _, s := range svcList.Items { if s.UID == svc.UID { continue @@ -842,7 +842,7 @@ func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.S if !r.isTailscaleService(&s) { continue } - if nameForService(&s) != svcName { + if reconciler.NameForService(&s) != svcName { continue } // Two ProxyGroups joined to different tailnets each have their own diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index eb39d0029..f00d9917b 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -24,20 +24,15 @@ "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" - "tailscale.com/net/dns/resolvconffile" "tailscale.com/tstime" "tailscale.com/util/clientmetric" - "tailscale.com/util/dnsname" "tailscale.com/util/set" ) const ( - resolvConfPath = "/etc/resolv.conf" - defaultClusterDomain = "cluster.local" - reasonProxyCreated = "ProxyCreated" reasonProxyInvalid = "ProxyInvalid" reasonProxyFailed = "ProxyFailed" @@ -156,7 +151,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len())) if !a.isTailscaleService(svc) { - tsoperator.RemoveServiceCondition(svc, tsapi.ProxyReady) + reconciler.RemoveServiceCondition(svc, tsapi.ProxyReady) } return nil } @@ -192,7 +187,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len())) if !a.isTailscaleService(svc) { - tsoperator.RemoveServiceCondition(svc, tsapi.ProxyReady) + reconciler.RemoveServiceCondition(svc, tsapi.ProxyReady) } return nil } @@ -218,14 +213,14 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga msg := fmt.Sprintf("unable to provision proxy resources: invalid config: %v", err) a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDCONFIG", msg) a.logger.Error(msg) - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger) return nil } - if violations := validateService(svc); len(violations) > 0 { + if violations := reconciler.ValidateService(svc); len(violations) > 0 { msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", ")) a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg) a.logger.Error(msg) - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger) return nil } @@ -233,11 +228,11 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga if proxyClass != "" { if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil { errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err) - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger) return errMsg } else if !ready { msg := fmt.Sprintf("ProxyClass %s specified for the Service, but is not (yet) Ready, waiting..", proxyClass) - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger) logger.Info(msg) return nil } @@ -252,7 +247,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga svc.Finalizers = append(svc.Finalizers, FinalizerName) if err := a.Update(ctx, svc); err != nil { errMsg := fmt.Errorf("failed to add finalizer: %w", err) - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger) return errMsg } } @@ -266,7 +261,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga Replicas: 1, ParentResourceName: svc.Name, ParentResourceUID: string(svc.UID), - Hostname: nameForService(svc), + Hostname: reconciler.NameForService(svc), Tags: tags, ChildResourceLabels: crl, ProxyClassName: proxyClass, @@ -304,12 +299,12 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga var hsvc *corev1.Service if hsvc, err = a.ssr.Provision(ctx, logger, sts); err != nil { errMsg := fmt.Errorf("failed to provision: %w", err) - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger) return errMsg } if sts.TailnetTargetIP != "" || sts.TailnetTargetFQDN != "" { // if an egress proxy - clusterDomain := retrieveClusterDomain(a.tsNamespace, logger) + clusterDomain := reconciler.ClusterDomain(a.tsNamespace, logger) headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc." + clusterDomain if svc.Spec.ExternalName != headlessSvcName || svc.Spec.Type != corev1.ServiceTypeExternalName { svc.Spec.ExternalName = headlessSvcName @@ -317,17 +312,17 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga svc.Spec.Type = corev1.ServiceTypeExternalName if err := a.Update(ctx, svc); err != nil { errMsg := fmt.Errorf("failed to update service: %w", err) - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger) return errMsg } } - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger) return nil } if !isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) { logger.Debugf("service is not a LoadBalancer, so not updating ingress") - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger) return nil } @@ -341,7 +336,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga logger.Debug(msg) // No hostname yet. Wait for the proxy pod to auth. svc.Status.LoadBalancer.Ingress = nil - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger) return nil } @@ -355,7 +350,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP) if err != nil { msg := fmt.Sprintf("failed to parse cluster IP: %v", err) - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger) return errors.New(msg) } @@ -370,44 +365,10 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga } svc.Status.LoadBalancer.Ingress = ingress - tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger) + reconciler.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger) return nil } -func validateService(svc *corev1.Service) []string { - violations := make([]string, 0) - if svc.Spec.ClusterIP == "None" { - violations = append(violations, "headless Services are not supported.") - } - if svc.Annotations[AnnotationTailnetTargetFQDN] != "" && svc.Annotations[AnnotationTailnetTargetIP] != "" { - violations = append(violations, fmt.Sprintf("only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN)) - } - if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" { - if !isMagicDNSName(fqdn) { - violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn)) - } - } - if ipStr := svc.Annotations[AnnotationTailnetTargetIP]; ipStr != "" { - ip, err := netip.ParseAddr(ipStr) - if err != nil { - violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q could not be parsed as a valid IP Address, error: %s", AnnotationTailnetTargetIP, ipStr, err)) - } else if !ip.IsValid() { - violations = append(violations, fmt.Sprintf("parsed IP address in annotation %s: %q is not valid", AnnotationTailnetTargetIP, ipStr)) - } - } - - svcName := nameForService(svc) - if err := dnsname.ValidLabel(svcName); err != nil { - if _, ok := svc.Annotations[AnnotationHostname]; ok { - violations = append(violations, fmt.Sprintf("invalid Tailscale hostname specified %q: %s", svcName, err)) - } else { - violations = append(violations, fmt.Sprintf("invalid Tailscale hostname %q, use %q annotation to override: %s", svcName, AnnotationHostname, err)) - } - } - violations = append(violations, tagViolations(svc)...) - return violations -} - func shouldExpose(svc *corev1.Service, isDefaultLoadBalancer bool) bool { return shouldExposeClusterIP(svc, isDefaultLoadBalancer) || shouldExposeDNSName(svc) } @@ -455,53 +416,5 @@ func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool if err := cl.Get(ctx, types.NamespacedName{Name: name}, proxyClass); err != nil { return false, fmt.Errorf("error getting ProxyClass %s: %w", name, err) } - return tsoperator.ProxyClassIsReady(proxyClass), nil -} - -// retrieveClusterDomain determines and retrieves cluster domain i.e -// (cluster.local) in which this Pod is running by parsing search domains in -// /etc/resolv.conf. If an error is encountered at any point during the process, -// defaults cluster domain to 'cluster.local'. -func retrieveClusterDomain(namespace string, logger *zap.SugaredLogger) string { - logger.Infof("attempting to retrieve cluster domain..") - conf, err := resolvconffile.ParseFile(resolvConfPath) - if err != nil { - // Vast majority of clusters use the cluster.local domain, so it - // is probably better to fall back to that than error out. - logger.Warn("error parsing /etc/resolv.conf to determine cluster domain, defaulting to 'cluster.local'.") - return defaultClusterDomain - } - return clusterDomainFromResolverConf(conf, namespace, logger) -} - -// clusterDomainFromResolverConf attempts to retrieve cluster domain from the provided resolver config. -// It expects the first three search domains in the resolver config to be ['.svc., svc., , ...] -// If the first three domains match the expected structure, it returns the third. -// If the domains don't match the expected structure or an error is encountered, it defaults to 'cluster.local' domain. -func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string, logger *zap.SugaredLogger) string { - if len(conf.SearchDomains) < 3 { - logger.Warnf(" resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.", len(conf.SearchDomains)) - return defaultClusterDomain - } - first := conf.SearchDomains[0] - if !strings.HasPrefix(string(first), namespace+".svc") { - logger.Warnf("first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.") - return defaultClusterDomain - } - second := conf.SearchDomains[1] - if !strings.HasPrefix(string(second), "svc") { - logger.Warnf("second search domain in resolver config is %s; expected 'svc.'.\nDefaulting cluster domain to 'cluster.local'.", second) - return defaultClusterDomain - } - // Trim the trailing dot for backwards compatibility purposes as the - // cluster domain was previously hardcoded to 'cluster.local' without a - // trailing dot. - probablyClusterDomain := strings.TrimPrefix(second.WithoutTrailingDot(), "svc.") - third := conf.SearchDomains[2] - if !strings.EqualFold(third.WithoutTrailingDot(), probablyClusterDomain) { - logger.Warnf("expected resolver config to contain serch domains .svc., svc., ; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third) - return defaultClusterDomain - } - logger.Infof("Cluster domain %q extracted from resolver config", probablyClusterDomain) - return probablyClusterDomain + return reconciler.ProxyClassIsReady(proxyClass), nil } diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index d46ebcb24..5ed0e07c4 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -664,21 +664,6 @@ func mustCreate(t *testing.T, client client.Client, obj client.Object) { t.Fatalf("creating %q: %v", obj.GetName(), err) } } -func mustCreateAll(t *testing.T, client client.Client, objs ...client.Object) { - t.Helper() - for _, obj := range objs { - mustCreate(t, client, obj) - } -} - -func mustDeleteAll(t *testing.T, client client.Client, objs ...client.Object) { - t.Helper() - for _, obj := range objs { - if err := client.Delete(context.Background(), obj); err != nil { - t.Fatalf("deleting %q: %v", obj.GetName(), err) - } - } -} func mustUpdate[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string, update func(O)) { t.Helper() @@ -985,19 +970,6 @@ func removeResourceReqs(sts *appsv1.StatefulSet) { } } -func removeTargetPortsFromSvc(svc *corev1.Service) { - newPorts := make([]corev1.ServicePort, 0) - for _, p := range svc.Spec.Ports { - newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port, Name: p.Name}) - } - svc.Spec.Ports = newPorts -} - -func removeClusterIPsFromSvc(svc *corev1.Service) { - svc.Spec.ClusterIP = "" - svc.Spec.ClusterIPs = nil -} - func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) { return func(secret *corev1.Secret) { t.Helper() diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go deleted file mode 100644 index 15fef5049..000000000 --- a/k8s-operator/conditions.go +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package kube - -import ( - "slices" - "time" - - "go.uber.org/zap" - xslices "golang.org/x/exp/slices" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - tsapi "tailscale.com/k8s-operator/apis/v1alpha1" - "tailscale.com/tstime" -) - -// SetConnectorCondition ensures that Connector status has a condition with the -// given attributes. LastTransitionTime gets set every time condition's status -// changes. -func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { - conds := updateCondition(cn.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) - cn.Status.Conditions = conds -} - -// RemoveConnectorCondition will remove condition of the given type if it exists. -func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConditionType) { - conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(conditionType) - }) -} - -// SetProxyClassCondition ensures that ProxyClass status has a condition with the -// given attributes. LastTransitionTime gets set every time condition's status -// changes. -func SetProxyClassCondition(pc *tsapi.ProxyClass, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { - conds := updateCondition(pc.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) - pc.Status.Conditions = conds -} - -// SetDNSConfigCondition ensures that DNSConfig status has a condition with the -// given attributes. LastTransitionTime gets set every time condition's status -// changes -func SetDNSConfigCondition(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { - conds := updateCondition(dnsCfg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) - dnsCfg.Status.Conditions = conds -} - -// SetServiceCondition ensures that Service status has a condition with the -// given attributes. LastTransitionTime gets set every time condition's status -// changes. -func SetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) { - conds := updateCondition(svc.Status.Conditions, conditionType, status, reason, message, 0, clock, logger) - svc.Status.Conditions = conds -} - -// GetServiceCondition returns Service condition with the specified type, if it exists on the Service. -func GetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) *metav1.Condition { - idx := xslices.IndexFunc(svc.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(conditionType) - }) - - if idx == -1 { - return nil - } - return &svc.Status.Conditions[idx] -} - -// RemoveServiceCondition will remove condition of the given type if it exists. -func RemoveServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) { - svc.Status.Conditions = slices.DeleteFunc(svc.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(conditionType) - }) -} - -// SetRecorderCondition ensures that Recorder status has a condition with the -// given attributes. LastTransitionTime gets set every time condition's status -// changes. -func SetRecorderCondition(tsr *tsapi.Recorder, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { - conds := updateCondition(tsr.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) - tsr.Status.Conditions = conds -} - -// SetProxyGroupCondition ensures that ProxyGroup status has a condition with the -// given attributes. LastTransitionTime gets set every time condition's status -// changes. -func SetProxyGroupCondition(pg *tsapi.ProxyGroup, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { - conds := updateCondition(pg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) - pg.Status.Conditions = conds -} - -// SetTailnetCondition ensures that Tailnet status has a condition with the -// given attributes. LastTransitionTime gets set every time condition's status -// changes. -func SetTailnetCondition(tn *tsapi.Tailnet, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) { - conds := updateCondition(tn.Status.Conditions, conditionType, status, reason, message, tn.Generation, clock, logger) - tn.Status.Conditions = conds -} - -// SetPeerRelayCondition ensures that PeerRelay status has a condition with the -// given attributes. LastTransitionTime gets set every time condition's status -// changes. -func SetPeerRelayCondition(pr *tsapi.PeerRelay, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) { - conds := updateCondition(pr.Status.Conditions, conditionType, status, reason, message, pr.Generation, clock, logger) - pr.Status.Conditions = conds -} - -func updateCondition(conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []metav1.Condition { - newCondition := metav1.Condition{ - Type: string(conditionType), - Status: status, - Reason: reason, - Message: message, - ObservedGeneration: gen, - } - - nowTime := metav1.NewTime(clock.Now().Truncate(time.Second)) - newCondition.LastTransitionTime = nowTime - - idx := xslices.IndexFunc(conds, func(cond metav1.Condition) bool { - return cond.Type == string(conditionType) - }) - - if idx == -1 { - conds = append(conds, newCondition) - return conds - } - - cond := conds[idx] // update the existing condition - - // If this update doesn't contain a state transition, don't update last - // transition time. - if cond.Status == status { - newCondition.LastTransitionTime = cond.LastTransitionTime - } else { - logger.Infof("Status change for condition %s from %s to %s", conditionType, cond.Status, status) - } - conds[idx] = newCondition - return conds -} - -func ProxyClassIsReady(pc *tsapi.ProxyClass) bool { - idx := xslices.IndexFunc(pc.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(tsapi.ProxyClassReady) - }) - if idx == -1 { - return false - } - cond := pc.Status.Conditions[idx] - return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pc.Generation -} - -func ProxyGroupIsReady(pg *tsapi.ProxyGroup) bool { - cond := proxyGroupCondition(pg, tsapi.ProxyGroupReady) - return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation -} - -func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool { - cond := proxyGroupCondition(pg, tsapi.ProxyGroupAvailable) - return cond != nil && cond.Status == metav1.ConditionTrue -} - -func KubeAPIServerProxyValid(pg *tsapi.ProxyGroup) (valid bool, set bool) { - cond := proxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid) - return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation, cond != nil -} - -func KubeAPIServerProxyConfigured(pg *tsapi.ProxyGroup) bool { - cond := proxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured) - return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation -} - -func proxyGroupCondition(pg *tsapi.ProxyGroup, condType tsapi.ConditionType) *metav1.Condition { - idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(condType) - }) - if idx == -1 { - return nil - } - return &pg.Status.Conditions[idx] -} - -func DNSCfgIsReady(cfg *tsapi.DNSConfig) bool { - idx := xslices.IndexFunc(cfg.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(tsapi.NameserverReady) - }) - if idx == -1 { - return false - } - cond := cfg.Status.Conditions[idx] - return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == cfg.Generation -} - -func SvcIsReady(svc *corev1.Service) bool { - idx := xslices.IndexFunc(svc.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(tsapi.ProxyReady) - }) - if idx == -1 { - return false - } - cond := svc.Status.Conditions[idx] - return cond.Status == metav1.ConditionTrue -} - -func TailnetIsReady(tn *tsapi.Tailnet) bool { - idx := xslices.IndexFunc(tn.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(tsapi.TailnetReady) - }) - if idx == -1 { - return false - } - cond := tn.Status.Conditions[idx] - return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == tn.Generation -} diff --git a/k8s-operator/conditions_test.go b/k8s-operator/conditions_test.go deleted file mode 100644 index 940a300d8..000000000 --- a/k8s-operator/conditions_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) Tailscale Inc & contributors -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package kube - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "go.uber.org/zap" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - tsapi "tailscale.com/k8s-operator/apis/v1alpha1" - "tailscale.com/tstest" -) - -func TestSetConnectorCondition(t *testing.T) { - cn := tsapi.Connector{} - clock := tstest.NewClock(tstest.ClockOpts{}) - fakeNow := metav1.NewTime(clock.Now().Truncate(time.Second)) - fakePast := metav1.NewTime(clock.Now().Truncate(time.Second).Add(-5 * time.Minute)) - zl, err := zap.NewDevelopment() - assert.Nil(t, err) - - // Set up a new condition - SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "someReason", "someMsg", 1, clock, zl.Sugar()) - assert.Equal(t, cn, tsapi.Connector{ - Status: tsapi.ConnectorStatus{ - Conditions: []metav1.Condition{ - { - Type: string(tsapi.ConnectorReady), - Status: metav1.ConditionTrue, - Reason: "someReason", - Message: "someMsg", - ObservedGeneration: 1, - LastTransitionTime: fakeNow, - }, - }, - }, - }) - - // Modify status of an existing condition - cn.Status = tsapi.ConnectorStatus{ - Conditions: []metav1.Condition{ - { - Type: string(tsapi.ConnectorReady), - Status: metav1.ConditionFalse, - Reason: "someReason", - Message: "someMsg", - ObservedGeneration: 1, - LastTransitionTime: fakePast, - }, - }, - } - SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar()) - assert.Equal(t, cn, tsapi.Connector{ - Status: tsapi.ConnectorStatus{ - Conditions: []metav1.Condition{ - { - Type: string(tsapi.ConnectorReady), - Status: metav1.ConditionTrue, - Reason: "anotherReason", - Message: "anotherMsg", - ObservedGeneration: 2, - LastTransitionTime: fakeNow, - }, - }, - }, - }) - - // Don't modify last transition time if status hasn't changed - cn.Status = tsapi.ConnectorStatus{ - Conditions: []metav1.Condition{ - { - Type: string(tsapi.ConnectorReady), - Status: metav1.ConditionTrue, - Reason: "someReason", - Message: "someMsg", - ObservedGeneration: 1, - LastTransitionTime: fakePast, - }, - }, - } - SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar()) - assert.Equal(t, cn, tsapi.Connector{ - Status: tsapi.ConnectorStatus{ - Conditions: []metav1.Condition{ - { - Type: string(tsapi.ConnectorReady), - Status: metav1.ConditionTrue, - Reason: "anotherReason", - Message: "anotherMsg", - ObservedGeneration: 2, - LastTransitionTime: fakePast, - }, - }, - }, - }) -} diff --git a/k8s-operator/reconciler/dnsrecords/dnsrecords.go b/k8s-operator/reconciler/dnsrecords/dnsrecords.go index 30daf87b9..5766e8457 100644 --- a/k8s-operator/reconciler/dnsrecords/dnsrecords.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords.go @@ -21,6 +21,7 @@ networkingv1 "k8s.io/api/networking/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/net" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -48,9 +49,6 @@ shortRequeue = 5 * time.Second - // AnnotationTailnetTargetFQDN is the annotation used to configure an egress proxy's tailnet target FQDN. - AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" - labelProxyGroup = "tailscale.com/proxy-group" labelSvcType = "tailscale.com/svc-type" typeEgress = "egress" @@ -150,7 +148,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res return reconcile.Result{}, nil } dnsCfg := dnsCfgLst.Items[0] - if !operatorutils.DNSCfgIsReady(&dnsCfg) { + if !nameserverReady(&dnsCfg) { logger.Info("DNSConfig is not ready yet, waiting...") return reconcile.Result{}, nil } @@ -343,7 +341,7 @@ func (r *Reconciler) fqdnForDNSRecord(ctx context.Context, proxySvc *corev1.Serv return "", err } - return svc.Annotations[AnnotationTailnetTargetFQDN], nil + return svc.Annotations[reconciler.AnnotationTailnetTargetFQDN], nil } return "", nil } @@ -391,7 +389,7 @@ func (r *Reconciler) isSvcForFQDNEgressProxy(ctx context.Context, svc *corev1.Se return false, err } annots := parentSvc.Annotations - return annots != nil && annots[AnnotationTailnetTargetFQDN] != "", nil + return annots != nil && annots[reconciler.AnnotationTailnetTargetFQDN] != "", nil } // isProxyGroupEgressService reports whether the Service is a ClusterIP Service @@ -435,7 +433,7 @@ func (r *Reconciler) parentSvcTargetsFQDN(ctx context.Context, svc *corev1.Servi if err := r.Get(ctx, parentName, parentSvc); err != nil { return false } - return parentSvc.Annotations[AnnotationTailnetTargetFQDN] != "" + return parentSvc.Annotations[reconciler.AnnotationTailnetTargetFQDN] != "" } // getTargetIPs returns the IPv4 and IPv6 addresses that should be used for DNS records @@ -621,3 +619,10 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z return reqs } } + +// nameserverReady reports whether the DNSConfig's nameserver is ready for the config's current generation, i.e. there +// is an in-cluster ts.net nameserver to write records for. +func nameserverReady(cfg *tsapi.DNSConfig) bool { + cond := reconciler.Condition(cfg.Status.Conditions, tsapi.NameserverReady) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == cfg.Generation +} diff --git a/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go b/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go index 43f7f0256..98d40a2ca 100644 --- a/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords_test.go @@ -83,7 +83,7 @@ func TestDNSRecordsReconciler(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) // Set the ready condition of the DNSConfig reconcilertest.MustUpdateStatus(t, fc, "", "test", func(c *tsapi.DNSConfig) { - operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, nameserver.ReasonNameserverCreated, nameserver.ReasonNameserverCreated, 0, cl, zl.Sugar()) + reconciler.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, nameserver.ReasonNameserverCreated, nameserver.ReasonNameserverCreated, 0, cl, zl.Sugar()) }) dnsRR := dnsrecords.NewReconciler(dnsrecords.ReconcilerOptions{ Client: fc, @@ -183,7 +183,7 @@ func TestDNSRecordsReconciler(t *testing.T) { Name: "external-service", Namespace: "default", Annotations: map[string]string{ - dnsrecords.AnnotationTailnetTargetFQDN: "external-service.example.ts.net", + reconciler.AnnotationTailnetTargetFQDN: "external-service.example.ts.net", }, }, Spec: corev1.ServiceSpec{ @@ -297,7 +297,7 @@ funcs := interceptor.Funcs{ Name: "lock-service", Namespace: "default", Annotations: map[string]string{ - dnsrecords.AnnotationTailnetTargetFQDN: "lock-service.example.ts.net", + reconciler.AnnotationTailnetTargetFQDN: "lock-service.example.ts.net", }, }, Spec: corev1.ServiceSpec{ @@ -433,7 +433,7 @@ func TestDNSRecordsReconcilerDualStack(t *testing.T) { Name: "pg-service", Namespace: "tailscale", Annotations: map[string]string{ - dnsrecords.AnnotationTailnetTargetFQDN: "pg-service.example.ts.net", + reconciler.AnnotationTailnetTargetFQDN: "pg-service.example.ts.net", }, }, Spec: corev1.ServiceSpec{ diff --git a/k8s-operator/reconciler/egress/egress.go b/k8s-operator/reconciler/egress/egress.go new file mode 100644 index 000000000..ba83cdd4a --- /dev/null +++ b/k8s-operator/reconciler/egress/egress.go @@ -0,0 +1,107 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package egress provides the reconcilers that expose tailnet targets to cluster workloads via an egress ProxyGroup. +// +// A user creates an ExternalName Service annotated with a tailnet target and a ProxyGroup. Four controllers then +// cooperate to make that target reachable: +// +// - Reconciler owns the user's ExternalName Service. It allocates a container port per Service port, creates a +// ClusterIP Service holding those portmappings, and writes the egress config the ProxyGroup's proxies read. +// - EndpointSliceReconciler keeps the ClusterIP Service's EndpointSlices pointing at the proxy Pods that are +// currently able to route traffic to the target. +// - ReadinessReconciler surfaces, on the user's Service, whether any proxy is actually ready to route to it. +// - PodReconciler gates proxy Pod readiness on the Pod having set up routing for its egress services, so that a +// rolling restart doesn't black-hole traffic. +// +// They are separate controllers rather than one because they watch different resources and must make progress +// independently; they share this package because they read and write the same egress config and labels. +package egress + +import ( + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/kube/kubetypes" +) + +const ( + // shortRequeue is how long to wait before retrying a reconcile that is waiting on state it doesn't control, e.g. + // a proxy Pod finishing its routing setup. + shortRequeue = 5 * time.Second + + // labelProxyGroup names the ProxyGroup that a managed resource belongs to. + labelProxyGroup = "tailscale.com/proxy-group" + + // labelSvcType distinguishes ingress from egress managed resources. + labelSvcType = "tailscale.com/svc-type" + typeEgress = "egress" + + // parentTypeSvc is the LabelParentType value used for resources owned by a user-created Service. + parentTypeSvc = "svc" + + // proxyTypeProxyGroup is the LabelParentType value carried by ProxyGroup-owned Pods. + proxyTypeProxyGroup = "proxygroup" + + // tsHealthCheckPortName names the port on the ClusterIP Service that proxies serve containerboot's health check + // endpoint on. It is excluded from the egress config, which only describes user-facing ports. + tsHealthCheckPortName = "tailscale-health-check" + + // maxPorts is the maximum number of ports that can be exposed on a + // container. In practice this will be ports in range [10000 - 11000). The + // high range should make it easier to distinguish container ports from + // the tailnet target ports for debugging purposes (i.e when reading + // netfilter rules). The limit of 1000 is somewhat arbitrary, the + // assumption is that this would not be hit in practice. + maxPorts = 1000 +) + +// CMName returns the name of the ConfigMap holding the egress service configs for the named ProxyGroup. The +// ProxyGroup reconciler creates it; the egress reconcilers read and update it. +func CMName(pg string) string { + return fmt.Sprintf("%s-egress-config", pg) +} + +// childResourceLabels returns the labels applied to the ClusterIP Service and EndpointSlices created for the egress +// service backing svc. +// +// TODO(irbekrm): we currently set a bunch of labels based on Kubernetes +// resource names (ProxyGroup, Service). Maximum allowed label length is 63 +// chars whilst the maximum allowed resource name length is 253 chars, so we +// should probably validate and truncate (?) the names is they are too long. +func childResourceLabels(svc *corev1.Service) map[string]string { + return map[string]string{ + kubetypes.LabelManaged: "true", + reconciler.LabelParentType: parentTypeSvc, + reconciler.LabelParentName: svc.Name, + reconciler.LabelParentNamespace: svc.Namespace, + labelProxyGroup: svc.Annotations[reconciler.AnnotationProxyGroup], + labelSvcType: typeEgress, + } +} + +// epsLabels returns the labels for an EndpointSlice created for an egress service, which are the child resource +// labels plus the two labels that make kube-proxy route ClusterIP Service traffic to this slice's endpoints. +func epsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string { + lbls := childResourceLabels(extNSvc) + // Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the + // endpoints defined on this EndpointSlice. + // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership + lbls[discoveryv1.LabelServiceName] = clusterIPSvc.Name + // Kubernetes recommends setting this label. + // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management + lbls[discoveryv1.LabelManagedBy] = "tailscale.com" + return lbls +} + +// tailnetSvcName returns the name used to distinguish the tailnet service exposed via extNSvc from the other tailnet +// services exposed to cluster workloads. It keys the egress config. +func tailnetSvcName(extNSvc *corev1.Service) string { + return fmt.Sprintf("%s-%s", extNSvc.Namespace, extNSvc.Name) +} diff --git a/k8s-operator/reconciler/egress/egress_test.go b/k8s-operator/reconciler/egress/egress_test.go new file mode 100644 index 000000000..53d7f58ee --- /dev/null +++ b/k8s-operator/reconciler/egress/egress_test.go @@ -0,0 +1,61 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package egress + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "tailscale.com/k8s-operator/reconciler" +) + +// The names and labels below are a contract with things outside this package: proxies select on the labels, the +// ProxyGroup reconciler creates the ConfigMap CMName refers to, and the egress config is keyed by tailnetSvcName. The +// rest of the package's tests build their fixtures from these helpers, so they would follow a change here rather than +// catch it; assert the literal values once. + +func TestCMName(t *testing.T) { + if got, want := CMName("pg"), "pg-egress-config"; got != want { + t.Errorf("CMName(\"pg\") = %q, want %q", got, want) + } +} + +func TestTailnetSvcName(t *testing.T) { + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "dev", Name: "my-app"}} + if got, want := tailnetSvcName(svc), "dev-my-app"; got != want { + t.Errorf("tailnetSvcName = %q, want %q", got, want) + } +} + +func TestChildResourceLabels(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "dev", + Name: "my-app", + Annotations: map[string]string{reconciler.AnnotationProxyGroup: "pg"}, + }, + } + + want := map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "svc", + "tailscale.com/parent-resource": "my-app", + "tailscale.com/parent-resource-ns": "dev", + "tailscale.com/proxy-group": "pg", + "tailscale.com/svc-type": "egress", + } + got := childResourceLabels(svc) + if len(got) != len(want) { + t.Fatalf("got %d labels, want %d: %v", len(got), len(want), got) + } + for k, v := range want { + if got[k] != v { + t.Errorf("label %q = %q, want %q", k, got[k], v) + } + } +} diff --git a/cmd/k8s-operator/egress-eps.go b/k8s-operator/reconciler/egress/endpointslices.go similarity index 77% rename from cmd/k8s-operator/egress-eps.go rename to k8s-operator/reconciler/egress/endpointslices.go index 1f02cfab9..b97a828bf 100644 --- a/cmd/k8s-operator/egress-eps.go +++ b/k8s-operator/reconciler/egress/endpointslices.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package egress import ( "context" @@ -18,24 +18,54 @@ discoveryv1 "k8s.io/api/discovery/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/egressservices" ) -// egressEpsReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies. -type egressEpsReconciler struct { +const endpointSliceReconcilerName = "egress-eps-reconciler" + +// EndpointSliceReconciler reconciles EndpointSlices for tailnet services exposed to cluster via egress ProxyGroup proxies. +type EndpointSliceReconciler struct { client.Client + logger *zap.SugaredLogger tsNamespace string } +// NewEndpointSliceReconciler returns the reconciler that keeps egress EndpointSlices pointing at the proxy Pods that +// can currently route to the tailnet target. +func NewEndpointSliceReconciler(opts Options) *EndpointSliceReconciler { + return &EndpointSliceReconciler{ + Client: opts.Client, + logger: opts.Logger.Named(endpointSliceReconcilerName), + tsNamespace: opts.TailscaleNamespace, + } +} + +// Register the EndpointSliceReconciler onto mgr. It watches the EndpointSlices it owns, the proxy Pods and their state +// Secrets (both of which determine whether a Pod can route), and the user's ExternalName Services. +func (er *EndpointSliceReconciler) Register(mgr manager.Manager) error { + return builder. + ControllerManagedBy(mgr). + Named(endpointSliceReconcilerName). + Watches(&discoveryv1.EndpointSlice{}, handler.EnqueueRequestsFromMapFunc(endpointSliceHandler)). + Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(endpointSlicesFromPods(er.Client, er.tsNamespace))). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(endpointSlicesFromStateSecrets(er.Client, er.tsNamespace))). + Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(endpointSlicesFromExternalNameService(er.Client, er.logger, er.tsNamespace))). + Complete(er) +} + // Reconcile reconciles an EndpointSlice for a tailnet service. It updates the EndpointSlice with the endpoints of // those ProxyGroup Pods that are ready to route traffic to the tailnet service. // It compares tailnet service state stored in egress proxy state Secrets by containerboot with the desired // configuration stored in proxy-cfg ConfigMap to determine if the endpoint is ready. -func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { +func (er *EndpointSliceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { lg := er.logger.With("Service", req.NamespacedName) lg.Debugf("starting reconcile") defer lg.Debugf("reconcile finished") @@ -58,8 +88,8 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ // resources are set up for this tailnet service. svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: eps.Labels[LabelParentName], - Namespace: eps.Labels[LabelParentNamespace], + Name: eps.Labels[reconciler.LabelParentName], + Namespace: eps.Labels[reconciler.LabelParentNamespace], }, } err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc) @@ -101,7 +131,7 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ // Check which Pods in ProxyGroup are ready to route traffic to this // egress service. podList := &corev1.PodList{} - if err := er.List(ctx, podList, client.MatchingLabels(pgLabels(proxyGroupName, nil))); err != nil { + if err := er.List(ctx, podList, client.MatchingLabels(reconciler.Labels("proxygroup", proxyGroupName, ""))); err != nil { return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err) } newEndpoints := make([]discoveryv1.Endpoint, 0) @@ -162,7 +192,7 @@ func podIPForFamily(pod *corev1.Pod, addrType discoveryv1.AddressType) (string, // podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to // route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service // status written there to the desired service configuration. -func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, addrType discoveryv1.AddressType, lg *zap.SugaredLogger) (bool, error) { +func (er *EndpointSliceReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, addrType discoveryv1.AddressType, lg *zap.SugaredLogger) (bool, error) { lg = lg.With("proxy_pod", pod.Name) lg.Debug("checking whether proxy is ready to route to egress service") if !pod.DeletionTimestamp.IsZero() { diff --git a/cmd/k8s-operator/egress-eps_test.go b/k8s-operator/reconciler/egress/endpointslices_test.go similarity index 66% rename from cmd/k8s-operator/egress-eps_test.go rename to k8s-operator/reconciler/egress/endpointslices_test.go index 14ed38d34..8cb875225 100644 --- a/cmd/k8s-operator/egress-eps_test.go +++ b/k8s-operator/reconciler/egress/endpointslices_test.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package egress import ( "encoding/json" @@ -17,7 +17,10 @@ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" @@ -32,8 +35,8 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { Namespace: "default", UID: types.UID("1234-UID"), Annotations: map[string]string{ - AnnotationTailnetTargetFQDN: "foo.bar.ts.net", - AnnotationProxyGroup: "foo", + reconciler.AnnotationTailnetTargetFQDN: "foo.bar.ts.net", + reconciler.AnnotationProxyGroup: "foo", }, }, Spec: corev1.ServiceSpec{ @@ -66,43 +69,43 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { if err != nil { t.Fatal(err) } - er := &egressEpsReconciler{ - Client: fc, - logger: zl.Sugar(), - tsNamespace: "operator-ns", - } + er := NewEndpointSliceReconciler(Options{ + Client: fc, + Logger: zl.Sugar(), + TailscaleNamespace: "operator-ns", + }) eps := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "operator-ns", Labels: map[string]string{ - LabelParentName: "test", - LabelParentNamespace: "default", - labelSvcType: typeEgress, - labelProxyGroup: "foo"}, + reconciler.LabelParentName: "test", + reconciler.LabelParentNamespace: "default", + labelSvcType: typeEgress, + labelProxyGroup: "foo"}, }, AddressType: discoveryv1.AddressTypeIPv4, } - mustCreate(t, fc, eps) + reconcilertest.MustCreate(t, fc, eps) t.Run("no_proxy_group_resources", func(t *testing.T) { - expectReconciled(t, er, "operator-ns", "foo") // should not error + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo") // should not error }) t.Run("no_pods_ready_to_route_traffic", func(t *testing.T) { pod, stateS := podAndSecretForProxyGroup("foo") - mustCreate(t, fc, pod) - mustCreate(t, fc, stateS) - expectReconciled(t, er, "operator-ns", "foo") // should not error + reconcilertest.MustCreate(t, fc, pod) + reconcilertest.MustCreate(t, fc, stateS) + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo") // should not error }) t.Run("pods_are_ready_to_route_traffic", func(t *testing.T) { pod, stateS := podAndSecretForProxyGroup("foo") stBs := serviceStatusForPodIPs(t, svc, pod.Status.PodIPs[0].IP, "", port) - mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) { + reconcilertest.MustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) { mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) }) - expectReconciled(t, er, "operator-ns", "foo") + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo") eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{ Addresses: []string{"10.0.0.1"}, Hostname: new("foo"), @@ -112,17 +115,17 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { Terminating: new(false), }, }) - expectEqual(t, fc, eps) + reconcilertest.ExpectEqual(t, fc, eps) }) t.Run("status_does_not_match_pod_ip", func(t *testing.T) { _, stateS := podAndSecretForProxyGroup("foo") // replica Pod has IP 10.0.0.1 stBs := serviceStatusForPodIPs(t, svc, "10.0.0.2", "", port) // status is for a Pod with IP 10.0.0.2 - mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) { + reconcilertest.MustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) { mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) }) - expectReconciled(t, er, "operator-ns", "foo") + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo") eps.Endpoints = []discoveryv1.Endpoint{} - expectEqual(t, fc, eps) + reconcilertest.ExpectEqual(t, fc, eps) }) // Dual-stack. @@ -131,34 +134,34 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { Name: "foo-ipv6", Namespace: "operator-ns", Labels: map[string]string{ - LabelParentName: "test", - LabelParentNamespace: "default", - labelSvcType: typeEgress, - labelProxyGroup: "foo", + reconciler.LabelParentName: "test", + reconciler.LabelParentNamespace: "default", + labelSvcType: typeEgress, + labelProxyGroup: "foo", }, }, AddressType: discoveryv1.AddressTypeIPv6, } - mustCreate(t, fc, epsV6) + reconcilertest.MustCreate(t, fc, epsV6) t.Run("dual_stack_pod_ready_to_route", func(t *testing.T) { - mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) + reconcilertest.MustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) dualPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo-0", Namespace: "operator-ns", - Labels: pgLabels("foo", nil), + Labels: reconciler.Labels("proxygroup", "foo", ""), UID: "foo", }, Status: corev1.PodStatus{ PodIPs: []corev1.PodIP{{IP: "10.0.0.1"}, {IP: "fd00::1"}}, }, } - mustCreate(t, fc, dualPod) + reconcilertest.MustCreate(t, fc, dualPod) stBs := serviceStatusForPodIPs(t, svc, "10.0.0.1", "fd00::1", port) - mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { + reconcilertest.MustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) }) - expectReconciled(t, er, "operator-ns", "foo") + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo") eps.Endpoints = []discoveryv1.Endpoint{{ Addresses: []string{"10.0.0.1"}, Hostname: new("foo"), @@ -168,8 +171,8 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { Terminating: new(false), }, }} - expectEqual(t, fc, eps) - expectReconciled(t, er, "operator-ns", "foo-ipv6") + reconcilertest.ExpectEqual(t, fc, eps) + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo-ipv6") epsV6.Endpoints = []discoveryv1.Endpoint{{ Addresses: []string{"fd00::1"}, Hostname: new("foo"), @@ -179,28 +182,28 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { Terminating: new(false), }, }} - expectEqual(t, fc, epsV6) + reconcilertest.ExpectEqual(t, fc, epsV6) }) // IPv6-only. t.Run("ipv4_only_pod_skipped_for_ipv6_slice", func(t *testing.T) { - mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) + reconcilertest.MustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) ipv4Pod, _ := podAndSecretForProxyGroup("foo") - mustCreate(t, fc, ipv4Pod) + reconcilertest.MustCreate(t, fc, ipv4Pod) stBs := serviceStatusForPodIPs(t, svc, "10.0.0.1", "", port) - mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { + reconcilertest.MustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) }) - expectReconciled(t, er, "operator-ns", "foo-ipv6") + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo-ipv6") // IPv4-only pod should not appear in the IPv6 EndpointSlice. epsV6.Endpoints = []discoveryv1.Endpoint{} - expectEqual(t, fc, epsV6) + reconcilertest.ExpectEqual(t, fc, epsV6) }) ipv6Pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "foo-0", Namespace: "operator-ns", - Labels: pgLabels("foo", nil), + Labels: reconciler.Labels("proxygroup", "foo", ""), UID: "foo", }, Status: corev1.PodStatus{ @@ -208,22 +211,22 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { }, } t.Run("ipv6_status_does_not_match_pod_ip", func(t *testing.T) { - mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) - mustCreate(t, fc, ipv6Pod) + reconcilertest.MustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}}) + reconcilertest.MustCreate(t, fc, ipv6Pod) stBs := serviceStatusForPodIPs(t, svc, "", "fd00::99", port) - mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { + reconcilertest.MustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) }) - expectReconciled(t, er, "operator-ns", "foo-ipv6") + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo-ipv6") epsV6.Endpoints = []discoveryv1.Endpoint{} - expectEqual(t, fc, epsV6) + reconcilertest.ExpectEqual(t, fc, epsV6) }) t.Run("ipv6_pod_ready_to_route", func(t *testing.T) { stBs := serviceStatusForPodIPs(t, svc, "", ipv6Pod.Status.PodIPs[0].IP, port) - mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { + reconcilertest.MustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) { mak.Set(&s.Data, egressservices.KeyEgressServices, stBs) }) - expectReconciled(t, er, "operator-ns", "foo-ipv6") + reconcilertest.ExpectReconciled(t, er, "operator-ns", "foo-ipv6") epsV6.Endpoints = append(epsV6.Endpoints, discoveryv1.Endpoint{ Addresses: []string{"fd00::1"}, Hostname: new("foo"), @@ -233,7 +236,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { Terminating: new(false), }, }) - expectEqual(t, fc, epsV6) + reconcilertest.ExpectEqual(t, fc, epsV6) }) } @@ -246,10 +249,10 @@ func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.Config cfg := egressservices.Config{ Ports: ports, } - if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" { + if fqdn := svc.Annotations[reconciler.AnnotationTailnetTargetFQDN]; fqdn != "" { cfg.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn} } - if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" { + if ip := svc.Annotations[reconciler.AnnotationTailnetTargetIP]; ip != "" { cfg.TailnetTarget = egressservices.TailnetTarget{IP: ip} } name := tailnetSvcName(svc) @@ -260,7 +263,7 @@ func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.Config } cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: pgEgressCMName(svc.Annotations[AnnotationProxyGroup]), + Name: CMName(svc.Annotations[reconciler.AnnotationProxyGroup]), Namespace: "operator-ns", }, BinaryData: map[string][]byte{egressservices.KeyEgressServices: bs}, @@ -275,10 +278,10 @@ func serviceStatusForPodIPs(t *testing.T, svc *corev1.Service, ipv4, ipv6 string ports[egressservices.PortMap{Protocol: string(port.Protocol), MatchPort: p, TargetPort: uint16(port.Port)}] = struct{}{} } svcSt := egressservices.ServiceStatus{Ports: ports} - if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" { + if fqdn := svc.Annotations[reconciler.AnnotationTailnetTargetFQDN]; fqdn != "" { svcSt.TailnetTarget = egressservices.TailnetTarget{FQDN: fqdn} } - if ip := svc.Annotations[AnnotationTailnetTargetIP]; ip != "" { + if ip := svc.Annotations[reconciler.AnnotationTailnetTargetIP]; ip != "" { svcSt.TailnetTarget = egressservices.TailnetTarget{IP: ip} } svcName := tailnetSvcName(svc) @@ -299,7 +302,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: pgLabels(pg, nil), + Labels: reconciler.Labels("proxygroup", pg, ""), UID: "foo", }, Status: corev1.PodStatus{ @@ -312,7 +315,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: pgSecretLabels(pg, kubetypes.LabelSecretTypeState), + Labels: stateSecretLabels(pg), }, } return p, s @@ -321,3 +324,11 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { func randomPort() uint16 { return uint16(rand.Int32N(1000) + 1000) } + +// stateSecretLabels returns the labels the ProxyGroup reconciler puts on a proxy's tailscaled state Secret. The egress +// reconcilers select on them to find which Pods are routing, so the fixtures have to match. +func stateSecretLabels(pgName string) map[string]string { + labels := reconciler.Labels("proxygroup", pgName, "") + labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeState + return labels +} diff --git a/k8s-operator/reconciler/egress/index.go b/k8s-operator/reconciler/egress/index.go new file mode 100644 index 000000000..5c0e64b78 --- /dev/null +++ b/k8s-operator/reconciler/egress/index.go @@ -0,0 +1,270 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package egress + +import ( + "context" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/kube/kubetypes" +) + +// IndexProxyGroup is the field index name under which egress Services are indexed by the ProxyGroup they are exposed +// on, so that a ProxyGroup event can enqueue every Service it serves. Reconciler.Register installs it. +const IndexProxyGroup = ".metadata.annotations.egress-proxy-group" + +// serviceHandler returns accepts a Kubernetes object and returns a reconcile +// request for it , if the object is a Tailscale egress Service meant to be +// exposed on a ProxyGroup. +func serviceHandler(_ context.Context, o client.Object) []reconcile.Request { + if !isEgressSvcForProxyGroup(o) { + return nil + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: o.GetNamespace(), + Name: o.GetName(), + }, + }, + } +} + +// servicesFromProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all +// user-created ExternalName Services that should be exposed on this ProxyGroup. +func servicesFromProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + pg, ok := o.(*tsapi.ProxyGroup) + if !ok { + logger.Warn("ProxyGroup handler triggered for an object that is not a ProxyGroup") + return nil + } + + if pg.Spec.Type != tsapi.ProxyGroupTypeEgress { + return nil + } + svcList := &corev1.ServiceList{} + if err := cl.List(ctx, svcList, client.MatchingFields{IndexProxyGroup: pg.Name}); err != nil { + logger.Infof("error listing Services: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name) + return nil + } + reqs := make([]reconcile.Request, 0) + for _, svc := range svcList.Items { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + }, + }) + } + return reqs + } +} + +// serviceFromEndpointSlice is an event handler for EndpointSlices. If an EndpointSlice is for an egress ExternalName Service +// meant to be exposed on a ProxyGroup, returns a reconcile request for the Service. +func serviceFromEndpointSlice(_ context.Context, o client.Object) []reconcile.Request { + if typ := o.GetLabels()[labelSvcType]; typ != typeEgress { + return nil + } + if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { + return nil + } + svcName, ok := o.GetLabels()[reconciler.LabelParentName] + if !ok { + return nil + } + svcNs, ok := o.GetLabels()[reconciler.LabelParentNamespace] + if !ok { + return nil + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: svcNs, + Name: svcName, + }, + }, + } +} + +// endpointSliceHandler returns accepts an EndpointSlice and, if the EndpointSlice +// is for an egress service, returns a reconcile request for it. +func endpointSliceHandler(_ context.Context, o client.Object) []reconcile.Request { + if typ := o.GetLabels()[labelSvcType]; typ != typeEgress { + return nil + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: o.GetNamespace(), + Name: o.GetName(), + }, + }, + } +} + +// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is, +// returns reconciler requests for all egress EndpointSlices for that ProxyGroup. +func endpointSlicesFromPods(cl client.Client, ns string) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { + return nil + } + // TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we + // have ingress ProxyGroups. + if typ := o.GetLabels()[reconciler.LabelParentType]; typ != "proxygroup" { + return nil + } + pg, ok := o.GetLabels()[reconciler.LabelParentName] + if !ok { + return nil + } + return endpointSliceRequestsForProxyGroup(ctx, pg, cl, ns) + } +} + +// endpointSlicesFromStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is, +// returns reconciler requests for all egress EndpointSlices for that ProxyGroup. +func endpointSlicesFromStateSecrets(cl client.Client, ns string) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { + return nil + } + if parentType := o.GetLabels()[reconciler.LabelParentType]; parentType != "proxygroup" { + return nil + } + if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState { + return nil + } + pg, ok := o.GetLabels()[reconciler.LabelParentName] + if !ok { + return nil + } + return endpointSliceRequestsForProxyGroup(ctx, pg, cl, ns) + } +} + +// endpointSlicesFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that +// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service. +func endpointSlicesFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + svc, ok := o.(*corev1.Service) + if !ok { + logger.Warn("Service handler triggered for an object that is not a Service") + return nil + } + + if !isEgressSvcForProxyGroup(svc) { + return nil + } + epsList := &discoveryv1.EndpointSliceList{} + if err := cl.List(ctx, epsList, client.InNamespace(ns), + client.MatchingLabels(childResourceLabels(svc))); err != nil { + logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name) + return nil + } + reqs := make([]reconcile.Request, 0) + for _, eps := range epsList.Items { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: eps.Namespace, + Name: eps.Name, + }, + }) + } + return reqs + } +} + +func podsFromEndpointSlices(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + eps, ok := o.(*discoveryv1.EndpointSlice) + if !ok { + logger.Warn("EndpointSlice handler triggered for an object that is not a EndpointSlice") + return nil + } + + if eps.Labels[labelProxyGroup] == "" { + return nil + } + if eps.Labels[labelSvcType] != "egress" { + return nil + } + podLabels := map[string]string{ + kubetypes.LabelManaged: "true", + reconciler.LabelParentType: "proxygroup", + reconciler.LabelParentName: eps.Labels[labelProxyGroup], + } + podList := &corev1.PodList{} + if err := cl.List(ctx, podList, client.InNamespace(ns), + client.MatchingLabels(podLabels)); err != nil { + logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on EndpointSlice %s", err, eps.Name) + return nil + } + reqs := make([]reconcile.Request, 0) + for _, pod := range podList.Items { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: pod.Namespace, + Name: pod.Name, + }, + }) + } + return reqs + } +} + +func podHandler(_ context.Context, o client.Object) []reconcile.Request { + if typ := o.GetLabels()[reconciler.LabelParentType]; typ != proxyTypeProxyGroup { + return nil + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: o.GetNamespace(), + Name: o.GetName(), + }, + }, + } +} + +// IndexServices adds a local index to cached Tailscale egress Services meant to be exposed on a ProxyGroup. The +// index is used a list filter. +func IndexServices(o client.Object) []string { + if !isEgressSvcForProxyGroup(o) { + return nil + } + return []string{o.GetAnnotations()[reconciler.AnnotationProxyGroup]} +} + +func endpointSliceRequestsForProxyGroup(ctx context.Context, pg string, cl client.Client, ns string) []reconcile.Request { + epsList := discoveryv1.EndpointSliceList{} + if err := cl.List(ctx, &epsList, + client.InNamespace(ns), + client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil { + return nil + } + reqs := make([]reconcile.Request, 0) + for _, ep := range epsList.Items { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: ep.Namespace, + Name: ep.Name, + }, + }) + } + return reqs +} diff --git a/cmd/k8s-operator/egress-pod-readiness.go b/k8s-operator/reconciler/egress/pods.go similarity index 79% rename from cmd/k8s-operator/egress-pod-readiness.go rename to k8s-operator/reconciler/egress/pods.go index acd68cb13..2ab463f22 100644 --- a/cmd/k8s-operator/egress-pod-readiness.go +++ b/k8s-operator/reconciler/egress/pods.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package egress import ( "context" @@ -17,32 +17,42 @@ "time" "go.uber.org/zap" - xslices "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" "tailscale.com/tstime" "tailscale.com/util/backoff" "tailscale.com/util/httpm" ) -const tsEgressReadinessGate = "tailscale.com/egress-services" +// ReadinessGate is the Pod readiness gate that PodReconciler sets once a proxy Pod has set up routing for its +// egress services. The ProxyGroup reconciler adds it to egress proxy Pod specs so that a rolling restart doesn't +// mark a Pod ready, and thus eligible for traffic, before it can actually route. +const ReadinessGate = "tailscale.com/egress-services" -// egressPodsReconciler is responsible for setting tailscale.com/egress-services condition on egress ProxyGroup Pods. +const podReconcilerName = "egress-pods-readiness-reconciler" + +// PodReconciler is responsible for setting tailscale.com/egress-services condition on egress ProxyGroup Pods. // The condition is used as a readiness gate for the Pod, meaning that kubelet will not mark the Pod as ready before the // condition is set. The ProxyGroup StatefulSet updates are rolled out in such a way that no Pod is restarted, before // the previous Pod is marked as ready, so ensuring that the Pod does not get marked as ready when it is not yet able to // route traffic for egress service prevents downtime during restarts caused by no available endpoints left because // every Pod has been recreated and is not yet added to endpoints. // https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-readiness-gate -type egressPodsReconciler struct { +type PodReconciler struct { client.Client + logger *zap.SugaredLogger tsNamespace string clock tstime.Clock @@ -50,6 +60,33 @@ type egressPodsReconciler struct { maxBackoff time.Duration // max backoff period between health check calls } +// NewPodReconciler returns the reconciler that gates egress proxy Pod readiness on the Pod having set up routing. +func NewPodReconciler(opts Options) *PodReconciler { + httpClient := opts.HTTPClient + if httpClient == nil { + httpClient = http.DefaultClient + } + return &PodReconciler{ + Client: opts.Client, + logger: opts.Logger.Named(podReconcilerName), + tsNamespace: opts.TailscaleNamespace, + clock: opts.clock(), + httpClient: httpClient, + maxBackoff: opts.MaxBackoff, + } +} + +// Register the PodReconciler onto mgr. It watches proxy Pods and the egress EndpointSlices that tell it which egress +// services a Pod is expected to be routing. +func (er *PodReconciler) Register(mgr manager.Manager) error { + return builder. + ControllerManagedBy(mgr). + Named(podReconcilerName). + Watches(&discoveryv1.EndpointSlice{}, handler.EnqueueRequestsFromMapFunc(podsFromEndpointSlices(er.Client, er.logger, er.tsNamespace))). + Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(podHandler)). + Complete(er) +} + // Reconcile reconciles an egress ProxyGroup Pods on changes to those Pods and ProxyGroup EndpointSlices. It ensures // that for each Pod who is ready to route traffic to all egress services for the ProxyGroup, the Pod has a // tailscale.com/egress-services condition to set, so that kubelet will mark the Pod as ready. @@ -72,7 +109,7 @@ type egressPodsReconciler struct { // // If the Pod does not appear to be serving the health check endpoint (pre-v1.80 proxies), the reconciler just sets the // readiness condition for backwards compatibility reasons. -func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { +func (er *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { lg := er.logger.With("Pod", req.NamespacedName) lg.Debugf("starting reconcile") defer lg.Debugf("reconcile finished") @@ -90,7 +127,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req return res, nil } - if pod.Labels[LabelParentType] != proxyTypeProxyGroup { + if pod.Labels[reconciler.LabelParentType] != proxyTypeProxyGroup { lg.Warn("reconciler called for a Pod that is not a ProxyGroup Pod") return res, nil } @@ -98,13 +135,13 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req // If the Pod does not have the readiness gate set, there is no need to add the readiness condition. In practice // this will happen if the user has configured custom TS_LOCAL_ADDR_PORT, thus disabling the graceful failover. if !slices.ContainsFunc(pod.Spec.ReadinessGates, func(r corev1.PodReadinessGate) bool { - return r.ConditionType == tsEgressReadinessGate + return r.ConditionType == ReadinessGate }) { lg.Debug("Pod does not have egress readiness gate set, skipping") return res, nil } - proxyGroupName := pod.Labels[LabelParentName] + proxyGroupName := pod.Labels[reconciler.LabelParentName] pg := new(tsapi.ProxyGroup) if err := er.Get(ctx, types.NamespacedName{Name: proxyGroupName}, pg); err != nil { return res, fmt.Errorf("error getting ProxyGroup %q: %w", proxyGroupName, err) @@ -126,10 +163,9 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req return res, fmt.Errorf("error listing ClusterIP Services") } - idx := xslices.IndexFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool { - return c.Type == tsEgressReadinessGate - }) - if idx != -1 { + if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool { + return c.Type == ReadinessGate + }) { lg.Debugf("Pod is already ready, do nothing") return res, nil } @@ -140,7 +176,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req s := svc go func() { ll := lg.With("service_name", s.Name) - d := retrieveClusterDomain(er.tsNamespace, ll) + d := reconciler.ClusterDomain(er.tsNamespace, ll) healthCheckAddr := healthCheckForSvc(&s, d) if healthCheckAddr == "" { ll.Debugf("ClusterIP Service does not expose a health check endpoint, unable to verify if routing is set up") @@ -150,7 +186,7 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req var routesSetup bool bo := backoff.NewBackoff(s.Name, ll.Infof, er.maxBackoff) - for range numCalls(pgReplicas(pg)) { + for range numCalls(reconciler.ProxyGroupReplicas(pg)) { if ctx.Err() != nil { errChan <- nil return @@ -192,15 +228,15 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req return res, nil } -func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, lg *zap.SugaredLogger) error { +func (er *PodReconciler) setPodReady(ctx context.Context, pod *corev1.Pod, lg *zap.SugaredLogger) error { if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool { - return c.Type == tsEgressReadinessGate + return c.Type == ReadinessGate }) { return nil } lg.Infof("Pod is ready to route traffic to all egress targets") pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ - Type: tsEgressReadinessGate, + Type: ReadinessGate, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Time{Time: er.clock.Now()}, }) @@ -221,7 +257,7 @@ func (er *egressPodsReconciler) setPodReady(ctx context.Context, pod *corev1.Pod ) // lookupPodRouteViaSvc attempts to reach a Pod using a health check endpoint served by a Service and returns the state of the health check. -func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, lg *zap.SugaredLogger) (healthCheckState, error) { +func (er *PodReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *corev1.Pod, healthCheckAddr string, lg *zap.SugaredLogger) (healthCheckState, error) { if !slices.ContainsFunc(pod.Spec.Containers[0].Env, func(e corev1.EnvVar) bool { return e.Name == "TS_ENABLE_HEALTH_CHECK" && e.Value == "true" }) { diff --git a/cmd/k8s-operator/egress-pod-readiness_test.go b/k8s-operator/reconciler/egress/pods_test.go similarity index 74% rename from cmd/k8s-operator/egress-pod-readiness_test.go rename to k8s-operator/reconciler/egress/pods_test.go index 6a087031b..eefa16ae5 100644 --- a/cmd/k8s-operator/egress-pod-readiness_test.go +++ b/k8s-operator/reconciler/egress/pods_test.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package egress import ( "bytes" @@ -21,7 +21,10 @@ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" ) @@ -35,12 +38,12 @@ func TestEgressPodReadiness(t *testing.T) { Build() zl, _ := zap.NewDevelopment() cl := tstest.NewClock(tstest.ClockOpts{}) - rec := &egressPodsReconciler{ - tsNamespace: "operator-ns", - Client: fc, - logger: zl.Sugar(), - clock: cl, - } + rec := NewPodReconciler(Options{ + TailscaleNamespace: "operator-ns", + Client: fc, + Logger: zl.Sugar(), + Clock: cl, + }) pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "dev", @@ -50,20 +53,20 @@ func TestEgressPodReadiness(t *testing.T) { Replicas: new(int32(3)), }, } - mustCreate(t, fc, pg) + reconcilertest.MustCreate(t, fc, pg) podIP := "10.0.0.2" podTemplate := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "operator-ns", Name: "pod", Labels: map[string]string{ - LabelParentType: "proxygroup", - LabelParentName: "dev", + reconciler.LabelParentType: "proxygroup", + reconciler.LabelParentName: "dev", }, }, Spec: corev1.PodSpec{ ReadinessGates: []corev1.PodReadinessGate{{ - ConditionType: tsEgressReadinessGate, + ConditionType: ReadinessGate, }}, Containers: []corev1.Container{{ Name: "tailscale", @@ -80,42 +83,42 @@ func TestEgressPodReadiness(t *testing.T) { t.Run("no_egress_services", func(t *testing.T) { pod := podTemplate.DeepCopy() - mustCreate(t, fc, pod) - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.MustCreate(t, fc, pod) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod) }) t.Run("one_svc_already_routed_to", func(t *testing.T) { pod := podTemplate.DeepCopy() svc, hep := newSvc("svc", 9002) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) resp := readyResps(podIP, 1) httpCl := fakeHTTPClient{ t: t, state: map[string][]fakeResponse{hep: resp}, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) + reconcilertest.ExpectEqual(t, fc, pod) // A subsequent reconcile should not change the Pod. - expectReconciled(t, rec, "operator-ns", pod.Name) - expectEqual(t, fc, pod) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) t.Run("one_svc_many_backends_eventually_routed_to", func(t *testing.T) { pod := podTemplate.DeepCopy() svc, hep := newSvc("svc", 9002) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) // For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only // return with the right Pod IP. resps := append(readyResps("10.0.0.3", 4), append(readyResps("10.0.0.4", 4), readyResps(podIP, 1)...)...) @@ -124,18 +127,18 @@ func TestEgressPodReadiness(t *testing.T) { state: map[string][]fakeResponse{hep: resps}, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) t.Run("one_svc_one_backend_eventually_healthy", func(t *testing.T) { pod := podTemplate.DeepCopy() svc, hep := newSvc("svc", 9002) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) // For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times, make the 9th time only // return with 200 status code. resps := append(unreadyResps(podIP, 8), readyResps(podIP, 1)...) @@ -144,18 +147,18 @@ func TestEgressPodReadiness(t *testing.T) { state: map[string][]fakeResponse{hep: resps}, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) t.Run("one_svc_one_backend_never_routable", func(t *testing.T) { pod := podTemplate.DeepCopy() svc, hep := newSvc("svc", 9002) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) // For a 3 replica ProxyGroup the healthcheck endpoint should be called 9 times and Pod should be // requeued if neither of those succeed. resps := readyResps("10.0.0.3", 9) @@ -164,11 +167,11 @@ func TestEgressPodReadiness(t *testing.T) { state: map[string][]fakeResponse{hep: resps}, } rec.httpClient = &httpCl - expectRequeue(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name) // Pod should not have readiness gate condition set. - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) t.Run("one_svc_many_backends_already_routable", func(t *testing.T) { pod := podTemplate.DeepCopy() @@ -176,7 +179,7 @@ func TestEgressPodReadiness(t *testing.T) { svc, hep := newSvc("svc", 9002) svc2, hep2 := newSvc("svc-2", 9002) svc3, hep3 := newSvc("svc-3", 9002) - mustCreateAll(t, fc, svc, svc2, svc3, pod) + reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod) resps := readyResps(podIP, 1) httpCl := fakeHTTPClient{ t: t, @@ -187,19 +190,19 @@ func TestEgressPodReadiness(t *testing.T) { }, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should not have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc, svc2, svc3) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3) }) t.Run("one_svc_many_backends_eventually_routable_and_healthy", func(t *testing.T) { pod := podTemplate.DeepCopy() svc, hep := newSvc("svc", 9002) svc2, hep2 := newSvc("svc-2", 9002) svc3, hep3 := newSvc("svc-3", 9002) - mustCreateAll(t, fc, svc, svc2, svc3, pod) + reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod) resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...) resps2 := append(readyResps("10.0.0.3", 5), readyResps(podIP, 1)...) resps3 := append(unreadyResps(podIP, 4), readyResps(podIP, 1)...) @@ -212,12 +215,12 @@ func TestEgressPodReadiness(t *testing.T) { }, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc, svc2, svc3) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3) }) t.Run("one_svc_many_backends_never_routable_and_healthy", func(t *testing.T) { pod := podTemplate.DeepCopy() @@ -225,7 +228,7 @@ func TestEgressPodReadiness(t *testing.T) { svc, hep := newSvc("svc", 9002) svc2, hep2 := newSvc("svc-2", 9002) svc3, hep3 := newSvc("svc-3", 9002) - mustCreateAll(t, fc, svc, svc2, svc3, pod) + reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod) // For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod // will be requeued if neither succeeds. resps := readyResps("10.0.0.3", 9) @@ -240,11 +243,11 @@ func TestEgressPodReadiness(t *testing.T) { }, } rec.httpClient = &httpCl - expectRequeue(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name) // Pod should not have readiness gate condition set. - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc, svc2, svc3) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3) }) t.Run("one_svc_many_backends_one_never_routable", func(t *testing.T) { pod := podTemplate.DeepCopy() @@ -252,7 +255,7 @@ func TestEgressPodReadiness(t *testing.T) { svc, hep := newSvc("svc", 9002) svc2, hep2 := newSvc("svc-2", 9002) svc3, hep3 := newSvc("svc-3", 9002) - mustCreateAll(t, fc, svc, svc2, svc3, pod) + reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod) // For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod // will be requeued if any one never succeeds. resps := readyResps(podIP, 9) @@ -267,11 +270,11 @@ func TestEgressPodReadiness(t *testing.T) { }, } rec.httpClient = &httpCl - expectRequeue(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name) // Pod should not have readiness gate condition set. - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc, svc2, svc3) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3) }) t.Run("one_svc_many_backends_one_never_healthy", func(t *testing.T) { pod := podTemplate.DeepCopy() @@ -279,7 +282,7 @@ func TestEgressPodReadiness(t *testing.T) { svc, hep := newSvc("svc", 9002) svc2, hep2 := newSvc("svc-2", 9002) svc3, hep3 := newSvc("svc-3", 9002) - mustCreateAll(t, fc, svc, svc2, svc3, pod) + reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod) // For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried 9 times and the Pod // will be requeued if any one never succeeds. resps := readyResps(podIP, 9) @@ -294,11 +297,11 @@ func TestEgressPodReadiness(t *testing.T) { }, } rec.httpClient = &httpCl - expectRequeue(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name) // Pod should not have readiness gate condition set. - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc, svc2, svc3) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3) }) t.Run("one_svc_many_backends_different_ports_eventually_healthy_and_routable", func(t *testing.T) { pod := podTemplate.DeepCopy() @@ -306,7 +309,7 @@ func TestEgressPodReadiness(t *testing.T) { svc, hep := newSvc("svc", 9003) svc2, hep2 := newSvc("svc-2", 9004) svc3, hep3 := newSvc("svc-3", 9010) - mustCreateAll(t, fc, svc, svc2, svc3, pod) + reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod) // For a ProxyGroup with 3 replicas, each Service's health endpoint will be tried up to 9 times and // marked as success as soon as one try succeeds. resps := append(readyResps("10.0.0.3", 7), readyResps(podIP, 1)...) @@ -321,12 +324,12 @@ func TestEgressPodReadiness(t *testing.T) { }, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc, svc2, svc3) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3) }) // Proxies of 1.78 and earlier did not set the Pod IP header. t.Run("pod_does_not_return_ip_header", func(t *testing.T) { @@ -334,7 +337,7 @@ func TestEgressPodReadiness(t *testing.T) { pod.Name = "foo-bar" svc, hep := newSvc("foo-bar", 9002) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) // If a response does not contain Pod IP header, we assume that this is an earlier proxy version, // readiness cannot be verified so the readiness gate is just set to true. resps := unreadyResps("", 1) @@ -345,18 +348,18 @@ func TestEgressPodReadiness(t *testing.T) { }, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) t.Run("one_svc_one_backend_eventually_healthy_and_routable", func(t *testing.T) { pod := podTemplate.DeepCopy() svc, hep := newSvc("svc", 9002) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) // If a response errors, it is probably because the Pod is not yet properly running, so retry. resps := append(erroredResps(8), readyResps(podIP, 1)...) httpCl := fakeHTTPClient{ @@ -366,12 +369,12 @@ func TestEgressPodReadiness(t *testing.T) { }, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) t.Run("one_svc_one_backend_svc_does_not_have_health_port", func(t *testing.T) { pod := podTemplate.DeepCopy() @@ -379,14 +382,14 @@ func TestEgressPodReadiness(t *testing.T) { // If a Service does not have health port set, we assume that it is not possible to determine Pod's // readiness and set it to ready. svc, _ := newSvc("svc", -1) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) rec.httpClient = nil - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) // Pod should have readiness gate condition set. podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) t.Run("error_setting_up_healthcheck", func(t *testing.T) { pod := podTemplate.DeepCopy() @@ -397,13 +400,13 @@ func TestEgressPodReadiness(t *testing.T) { svc, _ := newSvc("svc", 9002) svc2, _ := newSvc("svc-2", 9002) svc3, _ := newSvc("svc-3", 9002) - mustCreateAll(t, fc, svc, svc2, svc3, pod) + reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod) rec.httpClient = nil - expectError(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconcileError(t, rec, "operator-ns", pod.Name) // Pod should not have readiness gate condition set. - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc, svc2, svc3) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3) }) t.Run("pod_does_not_have_an_ip_address", func(t *testing.T) { pod := podTemplate.DeepCopy() @@ -412,38 +415,38 @@ func TestEgressPodReadiness(t *testing.T) { svc, _ := newSvc("svc", 9002) svc2, _ := newSvc("svc-2", 9002) svc3, _ := newSvc("svc-3", 9002) - mustCreateAll(t, fc, svc, svc2, svc3, pod) + reconcilertest.MustCreateAll(t, fc, svc, svc2, svc3, pod) rec.httpClient = nil - expectRequeue(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectRequeue(t, rec, "operator-ns", pod.Name) // Pod should not have readiness gate condition set. - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc, svc2, svc3) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc, svc2, svc3) }) t.Run("ipv6_only_pod_already_routed_to", func(t *testing.T) { pod := podTemplate.DeepCopy() pod.Status.PodIPs = []corev1.PodIP{{IP: "fd00::2"}} svc, hep := newSvc("svc", 9002) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) resp := readyRespsV6("fd00::2", 1) httpCl := fakeHTTPClient{ t: t, state: map[string][]fakeResponse{hep: resp}, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) t.Run("dual_stack_pod", func(t *testing.T) { pod := podTemplate.DeepCopy() pod.Status.PodIPs = []corev1.PodIP{{IP: "10.0.0.2"}, {IP: "fd00::2"}} svc, hep := newSvc("svc", 9002) - mustCreateAll(t, fc, svc, pod) + reconcilertest.MustCreateAll(t, fc, svc, pod) // Dual-stack pod: the reconciler uses PodIPs[0] (the primary IP), // which in this case is IPv4. resp := readyResps("10.0.0.2", 1) @@ -452,11 +455,11 @@ func TestEgressPodReadiness(t *testing.T) { state: map[string][]fakeResponse{hep: resp}, } rec.httpClient = &httpCl - expectReconciled(t, rec, "operator-ns", pod.Name) + reconcilertest.ExpectReconciled(t, rec, "operator-ns", pod.Name) podSetReady(pod, cl) - expectEqual(t, fc, pod) - mustDeleteAll(t, fc, pod, svc) + reconcilertest.ExpectEqual(t, fc, pod) + reconcilertest.MustDeleteAll(t, fc, pod, svc) }) } @@ -516,7 +519,7 @@ func newSvc(name string, port int32) (*corev1.Service, string) { func podSetReady(pod *corev1.Pod, cl *tstest.Clock) { pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ - Type: tsEgressReadinessGate, + Type: ReadinessGate, Status: corev1.ConditionTrue, LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, }) @@ -572,3 +575,12 @@ type fakeResponse struct { podIP string // for the Pod IP header header string // header key to use; defaults to PodIPv4Header } + +// TestReadinessGateName pins the readiness gate's wire value. The ProxyGroup reconciler stamps it onto egress proxy +// Pod specs and PodReconciler is what satisfies it, so the two must agree; every other test in this package builds its +// fixtures from the constant and so would follow a rename rather than catch it. +func TestReadinessGateName(t *testing.T) { + if got, want := ReadinessGate, "tailscale.com/egress-services"; got != want { + t.Errorf("ReadinessGate = %q, want %q", got, want) + } +} diff --git a/cmd/k8s-operator/egress-services-readiness.go b/k8s-operator/reconciler/egress/readiness.go similarity index 79% rename from cmd/k8s-operator/egress-services-readiness.go rename to k8s-operator/reconciler/egress/readiness.go index 76a8fdb21..d8dbd19f9 100644 --- a/cmd/k8s-operator/egress-services-readiness.go +++ b/k8s-operator/reconciler/egress/readiness.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package egress import ( "context" @@ -19,11 +19,14 @@ apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" "tailscale.com/tstime" "tailscale.com/util/set" ) @@ -38,18 +41,42 @@ msgReadyToRouteTemplate = "%d out of %d replicas are ready to route traffic" ) -type egressSvcsReadinessReconciler struct { +const readinessReconcilerName = "egress-svcs-readiness-reconciler" + +type ReadinessReconciler struct { client.Client + logger *zap.SugaredLogger clock tstime.Clock tsNamespace string } +// NewReadinessReconciler returns the reconciler that surfaces egress Service readiness. +func NewReadinessReconciler(opts Options) *ReadinessReconciler { + return &ReadinessReconciler{ + Client: opts.Client, + logger: opts.Logger.Named(readinessReconcilerName), + clock: opts.clock(), + tsNamespace: opts.TailscaleNamespace, + } +} + +// Register the ReadinessReconciler onto mgr. It watches egress Services and the EndpointSlices backing them, since +// readiness is derived from whether any endpoint is serving. +func (esrr *ReadinessReconciler) Register(mgr manager.Manager) error { + return builder. + ControllerManagedBy(mgr). + Named(readinessReconcilerName). + Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandler)). + Watches(&discoveryv1.EndpointSlice{}, handler.EnqueueRequestsFromMapFunc(serviceFromEndpointSlice)). + Complete(esrr) +} + // Reconcile reconciles an ExternalName Service that defines a tailnet target to be exposed on a ProxyGroup and sets the // EgressSvcReady condition on it. The condition gets set to true if at least one of the proxies is currently ready to // route traffic to the target. It compares proxy Pod IPs with the endpoints set on the EndpointSlice for the egress // service to determine how many replicas are currently able to route traffic. -func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { +func (esrr *ReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { lg := esrr.logger.With("Service", req.NamespacedName) lg.Debugf("starting reconcile") defer lg.Debugf("reconcile finished") @@ -67,13 +94,13 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re ) oldStatus := svc.Status.DeepCopy() defer func() { - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, lg) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, lg) if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) { err = errors.Join(err, esrr.Status().Update(ctx, svc)) } }() - crl := egressSvcChildResourceLabels(svc) + crl := childResourceLabels(svc) epsList := &discoveryv1.EndpointSliceList{} if err = esrr.List(ctx, epsList, client.InNamespace(esrr.tsNamespace), client.MatchingLabels(crl)); err != nil { err = fmt.Errorf("error listing EndpointSlices: %w", err) @@ -94,7 +121,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re // recreated when this status change re-triggers a Service reconcile. // // TODO(beckypauley): refactor so EndpointSlice recovery is not dependent on Service status. - clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esrr.Client, esrr.tsNamespace, crl) + clusterIPSvc, err := reconciler.GetSingleObject[corev1.Service](ctx, esrr.Client, esrr.tsNamespace, crl) if err != nil { err = fmt.Errorf("error retrieving ClusterIP Service: %w", err) reason = reasonReadinessCheckFailed @@ -127,7 +154,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re } pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ - Name: svc.Annotations[AnnotationProxyGroup], + Name: svc.Annotations[reconciler.AnnotationProxyGroup], }, } err = esrr.Get(ctx, client.ObjectKeyFromObject(pg), pg) @@ -143,26 +170,26 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re msg = err.Error() return res, err } - if !tsoperator.ProxyGroupAvailable(pg) { + if !reconciler.ProxyGroupAvailable(pg) { lg.Infof("ProxyGroup for Service is not ready, waiting...") reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady st = metav1.ConditionFalse return res, nil } - replicas := pgReplicas(pg) + replicas := reconciler.ProxyGroupReplicas(pg) if replicas == 0 { lg.Infof("ProxyGroup replicas set to 0") reason, msg = reasonNoProxies, reasonNoProxies st = metav1.ConditionFalse return res, nil } - podLabels := pgLabels(pg.Name, nil) + podLabels := reconciler.Labels("proxygroup", pg.Name, "") var readyReplicas int32 nextReplica: for i := range replicas { podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i) - pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels) + pod, err := reconciler.GetSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels) if err != nil { err = fmt.Errorf("error retrieving ProxyGroup Pod: %w", err) reason = reasonReadinessCheckFailed diff --git a/cmd/k8s-operator/egress-services-readiness_test.go b/k8s-operator/reconciler/egress/readiness_test.go similarity index 59% rename from cmd/k8s-operator/egress-services-readiness_test.go rename to k8s-operator/reconciler/egress/readiness_test.go index c21a111d5..a60d79217 100644 --- a/cmd/k8s-operator/egress-services-readiness_test.go +++ b/k8s-operator/reconciler/egress/readiness_test.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package egress import ( "fmt" @@ -15,8 +15,10 @@ discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" - tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/tstest" "tailscale.com/tstime" ) @@ -30,20 +32,20 @@ func TestEgressServiceReadiness(t *testing.T) { Build() zl, _ := zap.NewDevelopment() cl := tstest.NewClock(tstest.ClockOpts{}) - rec := &egressSvcsReadinessReconciler{ - tsNamespace: "operator-ns", - Client: fc, - logger: zl.Sugar(), - clock: cl, - } + rec := NewReadinessReconciler(Options{ + TailscaleNamespace: "operator-ns", + Client: fc, + Logger: zl.Sugar(), + Clock: cl, + }) tailnetFQDN := "my-app.tailnetxyz.ts.net" egressSvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "my-app", Namespace: "dev", Annotations: map[string]string{ - AnnotationProxyGroup: "dev", - AnnotationTailnetTargetFQDN: tailnetFQDN, + reconciler.AnnotationProxyGroup: "dev", + reconciler.AnnotationTailnetTargetFQDN: tailnetFQDN, }, }, } @@ -51,11 +53,11 @@ func TestEgressServiceReadiness(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "my-app", Namespace: "operator-ns", - Labels: egressSvcChildResourceLabels(egressSvc), + Labels: childResourceLabels(egressSvc), }, Spec: corev1.ServiceSpec{ClusterIPs: []string{"10.0.0.1"}}, } - labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) + labels := epsLabels(egressSvc, fakeClusterIPSvc) eps := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "my-app", @@ -69,59 +71,59 @@ func TestEgressServiceReadiness(t *testing.T) { Name: "dev", }, } - mustCreate(t, fc, egressSvc) - mustCreate(t, fc, fakeClusterIPSvc) + reconcilertest.MustCreate(t, fc, egressSvc) + reconcilertest.MustCreate(t, fc, fakeClusterIPSvc) setClusterNotReady(egressSvc, cl, zl.Sugar()) t.Run("endpointslice_does_not_exist", func(t *testing.T) { - expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc) // not ready + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + reconcilertest.ExpectEqual(t, fc, egressSvc) // not ready }) t.Run("proxy_group_does_not_exist", func(t *testing.T) { - mustCreate(t, fc, eps) - expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc) // still not ready + reconcilertest.MustCreate(t, fc, eps) + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + reconcilertest.ExpectEqual(t, fc, egressSvc) // still not ready }) t.Run("proxy_group_not_ready", func(t *testing.T) { - mustCreate(t, fc, pg) - expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc) // still not ready + reconcilertest.MustCreate(t, fc, pg) + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + reconcilertest.ExpectEqual(t, fc, egressSvc) // still not ready }) t.Run("no_ready_replicas", func(t *testing.T) { setPGReady(pg, cl, zl.Sugar()) - mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { + reconcilertest.MustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { p.Status = pg.Status }) - expectEqual(t, fc, pg) - for i := range pgReplicas(pg) { + reconcilertest.ExpectEqual(t, fc, pg) + for i := range reconciler.ProxyGroupReplicas(pg) { p := pod(pg, i) - mustCreate(t, fc, p) - mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) { + reconcilertest.MustCreate(t, fc, p) + reconcilertest.MustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) { existing.Status.PodIPs = p.Status.PodIPs }) } - expectReconciled(t, rec, "dev", "my-app") - setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg)) - expectEqual(t, fc, egressSvc) // still not ready + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + setNotReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg)) + reconcilertest.ExpectEqual(t, fc, egressSvc) // still not ready }) t.Run("one_ready_replica", func(t *testing.T) { setEndpointForReplica(pg, 0, eps) - mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) { e.Endpoints = eps.Endpoints }) - setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1) - expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc) // partially ready + setReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg), 1) + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + reconcilertest.ExpectEqual(t, fc, egressSvc) // partially ready }) t.Run("all_replicas_ready", func(t *testing.T) { - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { setEndpointForReplica(pg, i, eps) } - mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) { e.Endpoints = eps.Endpoints }) - setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg)) - expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc) // ready + setReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg), reconciler.ProxyGroupReplicas(pg)) + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + reconcilertest.ExpectEqual(t, fc, egressSvc) // ready }) } @@ -132,20 +134,20 @@ func TestEgressServiceReadinessDualStack(t *testing.T) { Build() zl, _ := zap.NewDevelopment() cl := tstest.NewClock(tstest.ClockOpts{}) - rec := &egressSvcsReadinessReconciler{ - tsNamespace: "operator-ns", - Client: fc, - logger: zl.Sugar(), - clock: cl, - } + rec := NewReadinessReconciler(Options{ + TailscaleNamespace: "operator-ns", + Client: fc, + Logger: zl.Sugar(), + Clock: cl, + }) tailnetFQDN := "my-app.tailnetxyz.ts.net" egressSvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "my-app", Namespace: "dev", Annotations: map[string]string{ - AnnotationProxyGroup: "dev", - AnnotationTailnetTargetFQDN: tailnetFQDN, + reconciler.AnnotationProxyGroup: "dev", + reconciler.AnnotationTailnetTargetFQDN: tailnetFQDN, }, }, } @@ -153,11 +155,11 @@ func TestEgressServiceReadinessDualStack(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "my-app", Namespace: "operator-ns", - Labels: egressSvcChildResourceLabels(egressSvc), + Labels: childResourceLabels(egressSvc), }, Spec: corev1.ServiceSpec{ClusterIPs: []string{"10.0.0.1", "fd00::1"}}, } - labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) + labels := epsLabels(egressSvc, fakeClusterIPSvc) epsV4 := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "my-app-ipv4", @@ -166,7 +168,7 @@ func TestEgressServiceReadinessDualStack(t *testing.T) { }, AddressType: discoveryv1.AddressTypeIPv4, } - labelsV6 := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) + labelsV6 := epsLabels(egressSvc, fakeClusterIPSvc) epsV6 := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "my-app-ipv6", @@ -184,41 +186,41 @@ func TestEgressServiceReadinessDualStack(t *testing.T) { Type: tsapi.ProxyGroupTypeEgress, }, } - mustCreate(t, fc, egressSvc) - mustCreate(t, fc, fakeClusterIPSvc) - mustCreate(t, fc, epsV4) - mustCreate(t, fc, epsV6) - mustCreate(t, fc, pg) + reconcilertest.MustCreate(t, fc, egressSvc) + reconcilertest.MustCreate(t, fc, fakeClusterIPSvc) + reconcilertest.MustCreate(t, fc, epsV4) + reconcilertest.MustCreate(t, fc, epsV6) + reconcilertest.MustCreate(t, fc, pg) setPGReady(pg, cl, zl.Sugar()) - mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { + reconcilertest.MustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { p.Status = pg.Status }) // Create a dual-stack pod. p := pod(pg, 0) p.Status.PodIPs = append(p.Status.PodIPs, corev1.PodIP{IP: "fd00::0"}) - mustCreate(t, fc, p) - mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) { + reconcilertest.MustCreate(t, fc, p) + reconcilertest.MustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) { existing.Status.PodIPs = p.Status.PodIPs }) t.Run("not_ready_missing_from_ipv6_slice", func(t *testing.T) { setEndpointForReplicaWithIP("10.0.0.0", epsV4) - mustUpdate(t, fc, epsV4.Namespace, epsV4.Name, func(e *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, epsV4.Namespace, epsV4.Name, func(e *discoveryv1.EndpointSlice) { e.Endpoints = epsV4.Endpoints }) - expectReconciled(t, rec, "dev", "my-app") - setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg)) - expectEqual(t, fc, egressSvc) + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + setNotReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg)) + reconcilertest.ExpectEqual(t, fc, egressSvc) }) t.Run("ready_in_both_slices", func(t *testing.T) { setEndpointForReplicaWithIP("fd00::", epsV6) - mustUpdate(t, fc, epsV6.Namespace, epsV6.Name, func(e *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, epsV6.Namespace, epsV6.Name, func(e *discoveryv1.EndpointSlice) { e.Endpoints = epsV6.Endpoints }) - expectReconciled(t, rec, "dev", "my-app") - setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg)) - expectEqual(t, fc, egressSvc) + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + setReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg), reconciler.ProxyGroupReplicas(pg)) + reconcilertest.ExpectEqual(t, fc, egressSvc) }) t.Run("not_ready_when_ipv6_slice_missing", func(t *testing.T) { // Delete the IPv6 EndpointSlice while the ClusterIP Service still @@ -227,9 +229,9 @@ func TestEgressServiceReadinessDualStack(t *testing.T) { if err := fc.Delete(t.Context(), epsV6); err != nil { t.Fatalf("error deleting IPv6 EndpointSlice: %v", err) } - expectReconciled(t, rec, "dev", "my-app") + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") setClusterNotReady(egressSvc, cl, zl.Sugar()) - expectEqual(t, fc, egressSvc) + reconcilertest.ExpectEqual(t, fc, egressSvc) }) } @@ -240,19 +242,19 @@ func TestEgressServiceReadinessIPv6Only(t *testing.T) { Build() zl, _ := zap.NewDevelopment() cl := tstest.NewClock(tstest.ClockOpts{}) - rec := &egressSvcsReadinessReconciler{ - tsNamespace: "operator-ns", - Client: fc, - logger: zl.Sugar(), - clock: cl, - } + rec := NewReadinessReconciler(Options{ + TailscaleNamespace: "operator-ns", + Client: fc, + Logger: zl.Sugar(), + Clock: cl, + }) egressSvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "my-app", Namespace: "dev", Annotations: map[string]string{ - AnnotationProxyGroup: "dev", - AnnotationTailnetTargetFQDN: "my-app.tailnetxyz.ts.net", + reconciler.AnnotationProxyGroup: "dev", + reconciler.AnnotationTailnetTargetFQDN: "my-app.tailnetxyz.ts.net", }, }, } @@ -260,11 +262,11 @@ func TestEgressServiceReadinessIPv6Only(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "my-app", Namespace: "operator-ns", - Labels: egressSvcChildResourceLabels(egressSvc), + Labels: childResourceLabels(egressSvc), }, Spec: corev1.ServiceSpec{ClusterIPs: []string{"fd00::1"}}, } - labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc) + labels := epsLabels(egressSvc, fakeClusterIPSvc) eps := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "my-app-ipv6", @@ -278,45 +280,45 @@ func TestEgressServiceReadinessIPv6Only(t *testing.T) { Name: "dev", }, } - mustCreate(t, fc, egressSvc) - mustCreate(t, fc, fakeClusterIPSvc) - mustCreate(t, fc, eps) - mustCreate(t, fc, pg) + reconcilertest.MustCreate(t, fc, egressSvc) + reconcilertest.MustCreate(t, fc, fakeClusterIPSvc) + reconcilertest.MustCreate(t, fc, eps) + reconcilertest.MustCreate(t, fc, pg) setPGReady(pg, cl, zl.Sugar()) - mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { + reconcilertest.MustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) { p.Status = pg.Status }) // Create IPv6-only pods. - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { p := ipv6OnlyPod(pg, i) - mustCreate(t, fc, p) - mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) { + reconcilertest.MustCreate(t, fc, p) + reconcilertest.MustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) { existing.Status.PodIPs = p.Status.PodIPs }) } t.Run("no_ready_replicas", func(t *testing.T) { - expectReconciled(t, rec, "dev", "my-app") - setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg)) - expectEqual(t, fc, egressSvc) + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + setNotReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg)) + reconcilertest.ExpectEqual(t, fc, egressSvc) }) t.Run("all_replicas_ready", func(t *testing.T) { - for i := range pgReplicas(pg) { + for i := range reconciler.ProxyGroupReplicas(pg) { p := ipv6OnlyPod(pg, i) setEndpointForReplicaWithIP(p.Status.PodIPs[0].IP, eps) } - mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) { + reconcilertest.MustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) { e.Endpoints = eps.Endpoints }) - setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg)) - expectReconciled(t, rec, "dev", "my-app") - expectEqual(t, fc, egressSvc) + setReady(egressSvc, cl, zl.Sugar(), reconciler.ProxyGroupReplicas(pg), reconciler.ProxyGroupReplicas(pg)) + reconcilertest.ExpectReconciled(t, rec, "dev", "my-app") + reconcilertest.ExpectEqual(t, fc, egressSvc) }) } func ipv6OnlyPod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod { - labels := pgLabels(pg.Name, nil) + labels := reconciler.Labels("proxygroup", pg.Name, "") labels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal) ip := fmt.Sprintf("fd00::%d", ordinal+1) // +1 to avoid fd00::0 normalization issues return &corev1.Pod{ @@ -332,12 +334,12 @@ func ipv6OnlyPod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod { } func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger) { - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, lg) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, lg) } func setNotReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger, replicas int32) { msg := fmt.Sprintf(msgReadyToRouteTemplate, 0, replicas) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, lg) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, lg) } func setReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger, replicas, readyReplicas int32) { @@ -346,11 +348,11 @@ func setReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger, repli reason = reasonReady } msg := fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, lg) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, lg) } func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, lg *zap.SugaredLogger) { - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, lg) + reconciler.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, lg) } func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) { @@ -366,7 +368,7 @@ func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1 } func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod { - labels := pgLabels(pg.Name, nil) + labels := reconciler.Labels("proxygroup", pg.Name, "") labels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal) ip := fmt.Sprintf("10.0.0.%d", ordinal) return &corev1.Pod{ diff --git a/cmd/k8s-operator/egress-services.go b/k8s-operator/reconciler/egress/services.go similarity index 77% rename from cmd/k8s-operator/egress-services.go rename to k8s-operator/reconciler/egress/services.go index f935fad64..32aa15dd0 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/k8s-operator/reconciler/egress/services.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package egress import ( "context" @@ -16,7 +16,7 @@ "reflect" "slices" "strings" - "sync" + "time" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -24,61 +24,105 @@ apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/storage/names" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/tailscaled" "tailscale.com/kube/egressservices" "tailscale.com/kube/kubetypes" "tailscale.com/tstime" "tailscale.com/util/clientmetric" "tailscale.com/util/mak" - "tailscale.com/util/set" ) const ( + reconcilerName = "egress-svcs-reconciler" + reasonEgressSvcInvalid = "EgressSvcInvalid" reasonEgressSvcValid = "EgressSvcValid" reasonEgressSvcCreationFailed = "EgressSvcCreationFailed" reasonProxyGroupNotReady = "ProxyGroupNotReady" - - labelProxyGroup = "tailscale.com/proxy-group" - - labelSvcType = "tailscale.com/svc-type" // ingress or egress - typeEgress = "egress" - // maxPorts is the maximum number of ports that can be exposed on a - // container. In practice this will be ports in range [10000 - 11000). The - // high range should make it easier to distinguish container ports from - // the tailnet target ports for debugging purposes (i.e when reading - // netfilter rules). The limit of 1000 is somewhat arbitrary, the - // assumption is that this would not be hit in practice. - maxPorts = 1000 - - indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group" - - tsHealthCheckPortName = "tailscale-health-check" ) var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount) -// egressSvcsReconciler reconciles user created ExternalName Services that specify a tailnet +// Reconciler reconciles user created ExternalName Services that specify a tailnet // endpoint that should be exposed to cluster workloads and an egress ProxyGroup // on whose proxies it should be exposed. -type egressSvcsReconciler struct { +type Reconciler struct { client.Client + logger *zap.SugaredLogger recorder record.EventRecorder clock tstime.Clock tsNamespace string - mu sync.Mutex // protects following - svcs set.Slice[types.UID] // UIDs of all currently managed egress Services for ProxyGroup + tracker *reconciler.ResourceTracker +} + +// Options contains configuration values shared by the egress reconcilers. +type Options struct { + // Client is used to interact with the Kubernetes API. + Client client.Client + // Recorder is used to emit Kubernetes events. Only Reconciler emits events. + Recorder record.EventRecorder + // TailscaleNamespace is the namespace the operator is installed in, where the ClusterIP Services, + // EndpointSlices and proxy Pods live. + TailscaleNamespace string + // Logger is the logger to use; each reconciler names a child logger after itself. + Logger *zap.SugaredLogger + // Clock is used to stamp condition transitions. Defaults to a real clock when unset. + Clock tstime.Clock + // HTTPClient is used by PodReconciler to call proxy health check endpoints. Defaults to + // http.DefaultClient when unset. + HTTPClient doer + // MaxBackoff caps the backoff between PodReconciler's health check calls. Zero means no cap, which is what + // the operator has always run with; tests set it to keep retries fast. + MaxBackoff time.Duration +} + +func (o Options) clock() tstime.Clock { + if o.Clock == nil { + return tstime.DefaultClock{} + } + return o.Clock +} + +// NewReconciler returns the reconciler for user-created egress ExternalName Services. +func NewReconciler(opts Options) *Reconciler { + return &Reconciler{ + Client: opts.Client, + recorder: opts.Recorder, + logger: opts.Logger.Named(reconcilerName), + clock: opts.clock(), + tsNamespace: opts.TailscaleNamespace, + tracker: reconciler.NewResourceTracker(gaugeEgressServices), + } +} + +// Register the Reconciler onto mgr. It watches egress Services directly and egress ProxyGroups so that a ProxyGroup +// becoming ready reconciles every Service exposed on it. It also installs the IndexProxyGroup field index that the +// ProxyGroup handler needs, so that callers can't forget to. +func (esr *Reconciler) Register(mgr manager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), IndexProxyGroup, IndexServices); err != nil { + return fmt.Errorf("failed to set up ProxyGroup indexer for egress Services: %w", err) + } + + return builder. + ControllerManagedBy(mgr). + Named(reconcilerName). + Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandler)). + Watches(&tsapi.ProxyGroup{}, handler.EnqueueRequestsFromMapFunc(servicesFromProxyGroup(esr.Client, esr.logger))). + Complete(esr) } // Reconcile reconciles an ExternalName Service that specifies a tailnet target and a ProxyGroup on whose proxies should @@ -99,7 +143,7 @@ type egressSvcsReconciler struct { // // - updates the egress service config in a ConfigMap mounted to the ProxyGroup proxies with the tailnet target and the // portmappings. -func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { +func (esr *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { lg := esr.logger.With("Service", req.NamespacedName) defer lg.Info("reconcile finished") @@ -139,29 +183,26 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re return res, nil } - if !slices.Contains(svc.Finalizers, FinalizerName) { - svc.Finalizers = append(svc.Finalizers, FinalizerName) + if !slices.Contains(svc.Finalizers, reconciler.Finalizer) { + svc.Finalizers = append(svc.Finalizers, reconciler.Finalizer) if err := esr.updateSvcSpec(ctx, svc); err != nil { err := fmt.Errorf("failed to add finalizer: %w", err) r := svcConfiguredReason(svc, false, lg) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg) return res, err } - esr.mu.Lock() - esr.svcs.Add(svc.UID) - gaugeEgressServices.Set(int64(esr.svcs.Len())) - esr.mu.Unlock() + esr.tracker.Add(svc.UID) } if err := esr.maybeCleanupProxyGroupConfig(ctx, svc, lg); err != nil { err = fmt.Errorf("cleaning up resources for previous ProxyGroup failed: %w", err) r := svcConfiguredReason(svc, false, lg) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, lg) return res, err } if err := esr.maybeProvision(ctx, svc, lg); err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { + if reconciler.IsOptimisticLockError(err) { lg.Infof("optimistic lock error, retrying: %s", err) } else { return reconcile.Result{}, err @@ -171,7 +212,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re return res, nil } -func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (err error) { +func (esr *Reconciler) maybeProvision(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (err error) { r := svcConfiguredReason(svc, false, lg) st := metav1.ConditionFalse defer func() { @@ -179,11 +220,11 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1 if st != metav1.ConditionTrue && err != nil { msg = err.Error() } - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, lg) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcConfigured, st, r, msg, esr.clock, lg) }() - crl := egressSvcChildResourceLabels(svc) - clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esr.Client, esr.tsNamespace, crl) + crl := childResourceLabels(svc) + clusterIPSvc, err := reconciler.GetSingleObject[corev1.Service](ctx, esr.Client, esr.tsNamespace, crl) if err != nil { err = fmt.Errorf("error retrieving ClusterIP Service: %w", err) return err @@ -194,7 +235,7 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1 upToDate := svcConfigurationUpToDate(svc, lg) provisioned := true if !upToDate { - if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[AnnotationProxyGroup], svc, clusterIPSvc, lg); err != nil { + if clusterIPSvc, provisioned, err = esr.provision(ctx, svc.Annotations[reconciler.AnnotationProxyGroup], svc, clusterIPSvc, lg); err != nil { return err } } @@ -208,7 +249,7 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1 } // Update ExternalName Service to point at the ClusterIP Service. - clusterDomain := retrieveClusterDomain(esr.tsNamespace, lg) + clusterDomain := reconciler.ClusterDomain(esr.tsNamespace, lg) clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain) if svc.Spec.ExternalName != clusterIPSvcFQDN { lg.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN) @@ -246,8 +287,8 @@ func addrTypesForClusterIPSvc(clusterIPSvc *corev1.Service) ([]discoveryv1.Addre // ensureEndpointSlices ensures that EndpointSlices exist for the egress service // for each IP family supported by the cluster, and that their ports are up to // date. -func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) error { - crl := egressSvcEpsLabels(svc, clusterIPSvc) +func (esr *Reconciler) ensureEndpointSlices(ctx context.Context, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) error { + crl := epsLabels(svc, clusterIPSvc) // Only create EndpointSlices for IP families supported by the cluster. addrTypes, err := addrTypesForClusterIPSvc(clusterIPSvc) if err != nil { @@ -263,7 +304,7 @@ func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc, AddressType: addrType, Ports: epsPortsFromSvc(clusterIPSvc), } - if _, err := createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) { + if _, err := reconciler.CreateOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) { e.Labels = eps.Labels e.AddressType = eps.AddressType e.Ports = eps.Ports @@ -277,7 +318,7 @@ func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc, return nil } -func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) (*corev1.Service, bool, error) { +func (esr *Reconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) (*corev1.Service, bool, error) { lg.Infof("updating configuration...") usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName) if err != nil { @@ -347,7 +388,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s }) } } - var healthCheckPort int32 = defaultLocalAddrPort + var healthCheckPort int32 = tailscaled.HealthCheckPort for { if !slices.ContainsFunc(svc.Spec.Ports, func(p corev1.ServicePort) bool { @@ -363,11 +404,11 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s clusterIPSvc.Spec.Ports = append(clusterIPSvc.Spec.Ports, corev1.ServicePort{ Name: tsHealthCheckPortName, Port: healthCheckPort, - TargetPort: intstr.FromInt(defaultLocalAddrPort), + TargetPort: intstr.FromInt(tailscaled.HealthCheckPort), Protocol: "TCP", }) if !reflect.DeepEqual(clusterIPSvc, oldClusterIPSvc) { - if clusterIPSvc, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) { + if clusterIPSvc, err = reconciler.CreateOrUpdate(ctx, esr.Client, esr.tsNamespace, clusterIPSvc, func(svc *corev1.Service) { svc.Labels = clusterIPSvc.Labels svc.Spec = clusterIPSvc.Spec }); err != nil { @@ -402,7 +443,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s return clusterIPSvc, true, nil } -func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error { +func (esr *Reconciler) maybeCleanup(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error { logger.Info("ensuring that resources created for egress service are deleted") // Delete egress service config from the ConfigMap mounted by the proxies. @@ -416,14 +457,14 @@ func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.S &corev1.Service{}, &discoveryv1.EndpointSlice{}, } - crl := egressSvcChildResourceLabels(svc) + crl := childResourceLabels(svc) for _, typ := range types { if err := esr.DeleteAllOf(ctx, typ, client.InNamespace(esr.tsNamespace), client.MatchingLabels(crl)); err != nil { return fmt.Errorf("error deleting %s: %w", typ, err) } } - ix := slices.Index(svc.Finalizers, FinalizerName) + ix := slices.Index(svc.Finalizers, reconciler.Finalizer) if ix != -1 { logger.Debug("Removing Tailscale finalizer from Service") svc.Finalizers = append(svc.Finalizers[:ix], svc.Finalizers[ix+1:]...) @@ -431,17 +472,14 @@ func (esr *egressSvcsReconciler) maybeCleanup(ctx context.Context, svc *corev1.S return fmt.Errorf("failed to remove finalizer: %w", err) } } - esr.mu.Lock() - esr.svcs.Remove(svc.UID) - gaugeEgressServices.Set(int64(esr.svcs.Len())) - esr.mu.Unlock() + esr.tracker.Remove(svc.UID) logger.Info("successfully cleaned up resources for egress Service") return nil } -func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) error { - wantsProxyGroup := svc.Annotations[AnnotationProxyGroup] - cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured) +func (esr *Reconciler) maybeCleanupProxyGroupConfig(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) error { + wantsProxyGroup := svc.Annotations[reconciler.AnnotationProxyGroup] + cond := reconciler.GetServiceCondition(svc, tsapi.EgressSvcConfigured) if cond == nil { return nil } @@ -468,7 +506,7 @@ func (esr *egressSvcsReconciler) maybeCleanupProxyGroupConfig(ctx context.Contex // latest ClusterIP Services via the controller cache. It will not work as well // once we split into multiple workers- at that point we probably want to set // used ports on ProxyGroup's status. -func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) (sets.Set[int32], error) { +func (esr *Reconciler) usedPortsForPG(ctx context.Context, pg string) (sets.Set[int32], error) { svcList := &corev1.ServiceList{} if err := esr.List(ctx, svcList, client.InNamespace(esr.tsNamespace), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil { return nil, fmt.Errorf("error listing Services: %w", err) @@ -486,10 +524,10 @@ func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) // for an egress service exposed on ProxyGroup proxies. The ClusterIP Service // has no selector. Traffic sent to it will be routed to the endpoints defined // by an EndpointSlice created for this egress service. -func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service { +func (esr *Reconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: svcNameBase(crl[LabelParentName]), + GenerateName: svcNameBase(crl[reconciler.LabelParentName]), Namespace: esr.tsNamespace, Labels: crl, }, @@ -500,9 +538,9 @@ func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *c } } -func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error { - crl := egressSvcChildResourceLabels(svc) - cmName := pgEgressCMName(crl[labelProxyGroup]) +func (esr *Reconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error { + crl := childResourceLabels(svc) + cmName := CMName(crl[labelProxyGroup]) cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: cmName, @@ -543,8 +581,8 @@ func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, return esr.Update(ctx, cm) } -func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (bool, error) { - proxyGroupName := svc.Annotations[AnnotationProxyGroup] +func (esr *Reconciler) validateClusterResources(ctx context.Context, svc *corev1.Service, lg *zap.SugaredLogger) (bool, error) { + proxyGroupName := svc.Annotations[reconciler.AnnotationProxyGroup] pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: proxyGroupName, @@ -552,35 +590,35 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s } if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) { lg.Infof("ProxyGroup %q not found, waiting...", proxyGroupName) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg) - tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg) + reconciler.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } else if err != nil { err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, lg) - tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, lg) + reconciler.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, err } if violations := validateEgressService(svc, pg); len(violations) > 0 { msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", ")) esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg) lg.Info(msg) - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, lg) - tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, lg) + reconciler.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } - if !tsoperator.ProxyGroupAvailable(pg) { - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg) - tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) + if !reconciler.ProxyGroupAvailable(pg) { + reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, lg) + reconciler.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) } lg.Debugf("egress service is valid") - tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, lg) + reconciler.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, lg) return true, nil } func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, lg *zap.SugaredLogger) egressservices.Config { - d := retrieveClusterDomain(ns, lg) + d := reconciler.ClusterDomain(ns, lg) tt := tailnetTargetFromSvc(externalNameSvc) hep := healthCheckForSvc(clusterIPSvc, d) cfg := egressservices.Config{ @@ -598,11 +636,11 @@ func egressSvcCfg(externalNameSvc, clusterIPSvc *corev1.Service, ns string, lg * } func validateEgressService(svc *corev1.Service, pg *tsapi.ProxyGroup) []string { - violations := validateService(svc) + violations := reconciler.ValidateService(svc) // We check that only one of these two is set in the earlier validateService function. - if svc.Annotations[AnnotationTailnetTargetFQDN] == "" && svc.Annotations[AnnotationTailnetTargetIP] == "" { - violations = append(violations, fmt.Sprintf("egress Service for ProxyGroup must have one of %s, %s annotations set", AnnotationTailnetTargetFQDN, AnnotationTailnetTargetIP)) + if svc.Annotations[reconciler.AnnotationTailnetTargetFQDN] == "" && svc.Annotations[reconciler.AnnotationTailnetTargetIP] == "" { + violations = append(violations, fmt.Sprintf("egress Service for ProxyGroup must have one of %s, %s annotations set", reconciler.AnnotationTailnetTargetFQDN, reconciler.AnnotationTailnetTargetIP)) } if len(svc.Spec.Ports) == 0 { violations = append(violations, "egress Service for ProxyGroup must have at least one target Port specified") @@ -656,13 +694,13 @@ func unusedPort(usedPorts sets.Set[int32]) int32 { // Service must contain exactly one of tailscale.com/tailnet-ip, // tailscale.com/tailnet-fqdn annotations. func tailnetTargetFromSvc(svc *corev1.Service) egressservices.TailnetTarget { - if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" { + if fqdn := svc.Annotations[reconciler.AnnotationTailnetTargetFQDN]; fqdn != "" { return egressservices.TailnetTarget{ FQDN: fqdn, } } return egressservices.TailnetTarget{ - IP: svc.Annotations[AnnotationTailnetTargetIP], + IP: svc.Annotations[reconciler.AnnotationTailnetTargetIP], } } @@ -681,13 +719,13 @@ func isEgressSvcForProxyGroup(obj client.Object) bool { return false } annots := s.ObjectMeta.Annotations - return annots[AnnotationProxyGroup] != "" && (annots[AnnotationTailnetTargetFQDN] != "" || annots[AnnotationTailnetTargetIP] != "") + return annots[reconciler.AnnotationProxyGroup] != "" && (annots[reconciler.AnnotationTailnetTargetFQDN] != "" || annots[reconciler.AnnotationTailnetTargetIP] != "") } // egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well // as unmarshalled configuration from the ConfigMap. func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs egressservices.Configs, err error) { - name := pgEgressCMName(proxyGroupName) + name := CMName(proxyGroupName) cm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -710,38 +748,8 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts return cm, cfgs, nil } -// egressSvcChildResourceLabels returns labels that should be applied to the -// ClusterIP Service and the EndpointSlice created for the egress service. -// TODO(irbekrm): we currently set a bunch of labels based on Kubernetes -// resource names (ProxyGroup, Service). Maximum allowed label length is 63 -// chars whilst the maximum allowed resource name length is 253 chars, so we -// should probably validate and truncate (?) the names is they are too long. -func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string { - return map[string]string{ - kubetypes.LabelManaged: "true", - LabelParentType: "svc", - LabelParentName: svc.Name, - LabelParentNamespace: svc.Namespace, - labelProxyGroup: svc.Annotations[AnnotationProxyGroup], - labelSvcType: typeEgress, - } -} - -// egressEpsLabels returns labels to be added to an EndpointSlice created for an egress service. -func egressSvcEpsLabels(extNSvc, clusterIPSvc *corev1.Service) map[string]string { - lbels := egressSvcChildResourceLabels(extNSvc) - // Adding this label is what makes kube proxy set up rules to route traffic sent to the clusterIP Service to the - // endpoints defined on this EndpointSlice. - // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership - lbels[discoveryv1.LabelServiceName] = clusterIPSvc.Name - // Kubernetes recommends setting this label. - // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#management - lbels[discoveryv1.LabelManagedBy] = "tailscale.com" - return lbels -} - func svcConfigurationUpToDate(svc *corev1.Service, lg *zap.SugaredLogger) bool { - cond := tsoperator.GetServiceCondition(svc, tsapi.EgressSvcConfigured) + cond := reconciler.GetServiceCondition(svc, tsapi.EgressSvcConfigured) if cond == nil { return false } @@ -781,23 +789,17 @@ func svcConfiguredReason(svc *corev1.Service, configured bool, lg *zap.SugaredLo } else { r = fmt.Sprintf("ConfigurationFailed:%s", r) } - r += fmt.Sprintf("ProxyGroup:%s", svc.Annotations[AnnotationProxyGroup]) + r += fmt.Sprintf("ProxyGroup:%s", svc.Annotations[reconciler.AnnotationProxyGroup]) tt := tailnetTargetFromSvc(svc) s := cfg{ Ports: svc.Spec.Ports, TailnetTarget: tt, - ProxyGroup: svc.Annotations[AnnotationProxyGroup], + ProxyGroup: svc.Annotations[reconciler.AnnotationProxyGroup], } r += fmt.Sprintf(":Config:%s", cfgHash(s, lg)) return r } -// tailnetSvc accepts and ExternalName Service name and returns a name that will be used to distinguish this tailnet -// service from other tailnet services exposed to cluster workloads. -func tailnetSvcName(extNSvc *corev1.Service) string { - return fmt.Sprintf("%s-%s", extNSvc.Namespace, extNSvc.Name) -} - // epsPortsFromSvc takes the ClusterIP Service created for an egress service and // returns its Port array in a form that can be used for an EndpointSlice. func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) { @@ -814,7 +816,7 @@ func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) { // updateSvcSpec ensures that the given Service's spec is updated in cluster, but the local Service object still retains // the not-yet-applied status. // TODO(irbekrm): once we do SSA for these patch updates, this will no longer be needed. -func (esr *egressSvcsReconciler) updateSvcSpec(ctx context.Context, svc *corev1.Service) error { +func (esr *Reconciler) updateSvcSpec(ctx context.Context, svc *corev1.Service) error { st := svc.Status.DeepCopy() err := esr.Update(ctx, svc) svc.Status = *st diff --git a/cmd/k8s-operator/egress-services_test.go b/k8s-operator/reconciler/egress/services_test.go similarity index 79% rename from cmd/k8s-operator/egress-services_test.go rename to k8s-operator/reconciler/egress/services_test.go index 20efe195d..314425540 100644 --- a/cmd/k8s-operator/egress-services_test.go +++ b/k8s-operator/reconciler/egress/services_test.go @@ -3,7 +3,7 @@ //go:build !plan9 -package main +package egress import ( "context" @@ -24,6 +24,8 @@ "sigs.k8s.io/controller-runtime/pkg/client/interceptor" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/reconcilertest" "tailscale.com/kube/egressservices" "tailscale.com/tstest" "tailscale.com/tstime" @@ -43,7 +45,7 @@ func TestTailscaleEgressServices(t *testing.T) { } cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: pgEgressCMName("foo"), + Name: CMName("foo"), Namespace: "operator-ns", }, } @@ -61,12 +63,12 @@ func TestTailscaleEgressServices(t *testing.T) { } clock := tstest.NewClock(tstest.ClockOpts{}) - esr := &egressSvcsReconciler{ - Client: fc, - logger: zl.Sugar(), - clock: clock, - tsNamespace: "operator-ns", - } + esr := NewReconciler(Options{ + Client: fc, + Logger: zl.Sugar(), + Clock: clock, + TailscaleNamespace: "operator-ns", + }) tailnetTargetFQDN := "foo.bar.ts.net." svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -74,8 +76,8 @@ func TestTailscaleEgressServices(t *testing.T) { Namespace: "default", UID: types.UID("1234-UID"), Annotations: map[string]string{ - AnnotationTailnetTargetFQDN: tailnetTargetFQDN, - AnnotationProxyGroup: "foo", + reconciler.AnnotationTailnetTargetFQDN: tailnetTargetFQDN, + reconciler.AnnotationProxyGroup: "foo", }, }, Spec: corev1.ServiceSpec{ @@ -92,32 +94,32 @@ func TestTailscaleEgressServices(t *testing.T) { } t.Run("service_one_unnamed_port", func(t *testing.T) { - mustCreate(t, fc, svc) - expectReconciled(t, esr, "default", "test") + reconcilertest.MustCreate(t, fc, svc) + reconcilertest.ExpectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("service_add_two_named_ports", func(t *testing.T) { svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}} - mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "default", "test", func(s *corev1.Service) { s.Spec.Ports = svc.Spec.Ports }) - expectReconciled(t, esr, "default", "test") + reconcilertest.ExpectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("service_add_udp_port", func(t *testing.T) { svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Port: 53, Protocol: "UDP", Name: "dns"}) - mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "default", "test", func(s *corev1.Service) { s.Spec.Ports = svc.Spec.Ports }) - expectReconciled(t, esr, "default", "test") + reconcilertest.ExpectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("service_change_protocol", func(t *testing.T) { svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}, {Port: 53, Protocol: "TCP", Name: "tcp_dns"}} - mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { + reconcilertest.MustUpdate(t, fc, "default", "test", func(s *corev1.Service) { s.Spec.Ports = svc.Spec.Ports }) - expectReconciled(t, esr, "default", "test") + reconcilertest.ExpectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) @@ -134,7 +136,7 @@ func TestTailscaleEgressServices(t *testing.T) { if err := fc.Delete(t.Context(), eps); err != nil { t.Fatalf("error deleting EndpointSlice: %v", err) } - expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) + reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) validateReadyService(t, fc, esr, svc, clock, zl, cm) }) @@ -143,23 +145,23 @@ func TestTailscaleEgressServices(t *testing.T) { if err := fc.Delete(context.Background(), svc); err != nil { t.Fatalf("error deleting ExternalName Service: %v", err) } - expectReconciled(t, esr, "default", "test") + reconcilertest.ExpectReconciled(t, esr, "default", "test") // Verify that ClusterIP Service and EndpointSlice have been deleted. - expectMissing[corev1.Service](t, fc, "operator-ns", name) - expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name)) + reconcilertest.ExpectMissing[corev1.Service](t, fc, "operator-ns", name) + reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name)) // Verify that service config has been deleted from the ConfigMap. mustNotHaveConfigForSvc(t, fc, svc, cm) }) } -func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReconciler, svc *corev1.Service, clock *tstest.Clock, zl *zap.Logger, cm *corev1.ConfigMap) { - expectReconciled(t, esr, "default", "test") +func validateReadyService(t *testing.T, fc client.WithWatch, esr *Reconciler, svc *corev1.Service, clock *tstest.Clock, zl *zap.Logger, cm *corev1.ConfigMap) { + reconcilertest.ExpectReconciled(t, esr, "default", "test") // Verify that a ClusterIP Service has been created. name := findGenNameForEgressSvcResources(t, fc, svc) - expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc, removeClusterIPsFromSvc) + reconcilertest.ExpectEqual(t, fc, clusterIPSvc(name, svc), reconcilertest.RemoveTargetPorts, reconcilertest.RemoveClusterIPs) clusterSvc := mustGetClusterIPSvc(t, fc, name) // Verify that an EndpointSlice has been created. - expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4)) + reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4)) // Verify that ConfigMap contains configuration for the new egress service. mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm, zl) r := svcConfiguredReason(svc, true, zl.Sugar()) @@ -171,7 +173,7 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco } svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"} svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name) - expectEqual(t, fc, svc) + reconcilertest.ExpectEqual(t, fc, svc) } @@ -179,7 +181,7 @@ func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string return metav1.Condition{ Type: string(typ), Status: st, - LastTransitionTime: conditionTime(clock), + LastTransitionTime: reconcilertest.ConditionTime(clock), Reason: r, Message: msg, } @@ -187,8 +189,8 @@ func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string { t.Helper() - labels := egressSvcChildResourceLabels(svc) - s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels) + labels := childResourceLabels(svc) + s, err := reconciler.GetSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels) if err != nil { t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err) } @@ -199,7 +201,7 @@ func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *c } func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service { - labels := egressSvcChildResourceLabels(extNSvc) + labels := childResourceLabels(extNSvc) ports := make([]corev1.ServicePort, len(extNSvc.Spec.Ports)) for i, port := range extNSvc.Spec.Ports { ports[i] = corev1.ServicePort{ // Copy the port to avoid modifying the original. @@ -246,7 +248,7 @@ func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Se } func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service, addrType discoveryv1.AddressType) *discoveryv1.EndpointSlice { - labels := egressSvcChildResourceLabels(extNSvc) + labels := childResourceLabels(extNSvc) labels[discoveryv1.LabelManagedBy] = "tailscale.com" labels[discoveryv1.LabelServiceName] = name suffix := "ipv4" @@ -335,7 +337,7 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) { } cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: pgEgressCMName("foo"), + Name: CMName("foo"), Namespace: "operator-ns", }, } @@ -353,20 +355,20 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) { } clock := tstest.NewClock(tstest.ClockOpts{}) - esr := &egressSvcsReconciler{ - Client: fc, - logger: zl.Sugar(), - clock: clock, - tsNamespace: "operator-ns", - } + esr := NewReconciler(Options{ + Client: fc, + Logger: zl.Sugar(), + Clock: clock, + TailscaleNamespace: "operator-ns", + }) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", UID: types.UID("1234-UID"), Annotations: map[string]string{ - AnnotationTailnetTargetFQDN: "foo.bar.ts.net.", - AnnotationProxyGroup: "foo", + reconciler.AnnotationTailnetTargetFQDN: "foo.bar.ts.net.", + reconciler.AnnotationProxyGroup: "foo", }, }, Spec: corev1.ServiceSpec{ @@ -383,13 +385,13 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) { } t.Run("dual_stack_creates_both_endpoint_slices", func(t *testing.T) { - mustCreate(t, fc, svc) - expectReconciled(t, esr, "default", "test") + reconcilertest.MustCreate(t, fc, svc) + reconcilertest.ExpectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) // Also verify the IPv6 EndpointSlice was created. name := findGenNameForEgressSvcResources(t, fc, svc) clusterSvc := mustGetClusterIPSvc(t, fc, name) - expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) + reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) }) t.Run("dual_stack_endpointslice_deletion_recovery", func(t *testing.T) { @@ -406,12 +408,12 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) { if err := fc.Delete(t.Context(), eps); err != nil { t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err) } - expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) + reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) } // Reconcile should recreate both. validateReadyService(t, fc, esr, svc, clock, zl, cm) clusterSvc := mustGetClusterIPSvc(t, fc, name) - expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) + reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) }) t.Run("dual_stack_single_endpointslice_deletion_recovery", func(t *testing.T) { @@ -427,13 +429,13 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) { if err := fc.Delete(t.Context(), eps); err != nil { t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err) } - expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) + reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) // Reconcile should recreate the missing IPv6 EndpointSlice while leaving // the IPv4 one untouched. validateReadyService(t, fc, esr, svc, clock, zl, cm) clusterSvc := mustGetClusterIPSvc(t, fc, name) - expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) - expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4)) + reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) + reconcilertest.ExpectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4)) }) t.Run("delete_dual_stack_service", func(t *testing.T) { @@ -441,10 +443,10 @@ func TestTailscaleEgressServicesDualStack(t *testing.T) { if err := fc.Delete(context.Background(), svc); err != nil { t.Fatalf("error deleting ExternalName Service: %v", err) } - expectReconciled(t, esr, "default", "test") - expectMissing[corev1.Service](t, fc, "operator-ns", name) - expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name)) - expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv6", name)) + reconcilertest.ExpectReconciled(t, esr, "default", "test") + reconcilertest.ExpectMissing[corev1.Service](t, fc, "operator-ns", name) + reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name)) + reconcilertest.ExpectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv6", name)) mustNotHaveConfigForSvc(t, fc, svc, cm) }) } diff --git a/k8s-operator/reconciler/nameserver/nameserver.go b/k8s-operator/reconciler/nameserver/nameserver.go index 705b7e177..b9f030f62 100644 --- a/k8s-operator/reconciler/nameserver/nameserver.go +++ b/k8s-operator/reconciler/nameserver/nameserver.go @@ -27,7 +27,6 @@ "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" @@ -132,7 +131,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res oldCnStatus := dnsCfg.Status.DeepCopy() setStatus := func(dnsCfg *tsapi.DNSConfig, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, r.clock, logger) + reconciler.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, r.clock, logger) if !apiequality.Semantic.DeepEqual(oldCnStatus, &dnsCfg.Status) { // An error encountered here should get returned by the Reconcile function. if updateErr := r.Client.Status().Update(ctx, dnsCfg); updateErr != nil { diff --git a/k8s-operator/reconciler/peerrelay/peerrelay.go b/k8s-operator/reconciler/peerrelay/peerrelay.go index f2e5c46ac..27ec01e96 100644 --- a/k8s-operator/reconciler/peerrelay/peerrelay.go +++ b/k8s-operator/reconciler/peerrelay/peerrelay.go @@ -31,7 +31,6 @@ "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/tailscaled" @@ -201,7 +200,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco } func (r *Reconciler) reportTailnetUnavailable(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, tsErr error) (reconcile.Result, error) { - operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonTailnetUnavailable, tsErr.Error(), r.clock, logger) + setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonTailnetUnavailable, tsErr.Error(), r.clock, logger) if err := r.Status().Update(ctx, pr); err != nil { return reconcile.Result{}, errors.Join(tsErr, fmt.Errorf("failed to update PeerRelay status: %w", err)) } @@ -229,7 +228,7 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, logger *zap.SugaredLogg // condition so they can fix the spec. if pr.Spec.AWS != nil && int32(len(pr.Spec.AWS.ElasticIPs)) < replicas { message := fmt.Sprintf("spec.aws.elasticIPs has %d entries but spec.replicas is %d", len(pr.Spec.AWS.ElasticIPs), replicas) - operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonAWSConfigInvalid, message, r.clock, logger) + setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonAWSConfigInvalid, message, r.clock, logger) if err := r.Status().Update(ctx, pr); err != nil { return reconcile.Result{}, fmt.Errorf("failed to update PeerRelay status for %q: %w", pr.Name, err) } @@ -358,12 +357,12 @@ func (r *Reconciler) writeStatus(ctx context.Context, logger *zap.SugaredLogger, switch { case int32(len(addressed)) < replicas: message := fmt.Sprintf("%d of %d replicas have a public IP", len(addressed), replicas) - operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonEndpointsPending, message, r.clock, logger) + setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonEndpointsPending, message, r.clock, logger) case readyReplicas < replicas: message := fmt.Sprintf("%d of %d pods are ready", readyReplicas, replicas) - operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonPodsPending, message, r.clock, logger) + setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonPodsPending, message, r.clock, logger) default: - operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionTrue, ReasonReady, ReasonReady, r.clock, logger) + setCondition(pr, tsapi.PeerRelayReady, metav1.ConditionTrue, ReasonReady, ReasonReady, r.clock, logger) } if reflect.DeepEqual(prevStatus, &pr.Status) { @@ -531,3 +530,9 @@ func (r *Reconciler) deleteStatefulSet(ctx context.Context, logger *zap.SugaredL } return nil } + +// setCondition sets a condition on pr's status. ObservedGeneration is always the PeerRelay's own generation, so +// callers don't pass it. +func setCondition(pr *tsapi.PeerRelay, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) { + pr.Status.Conditions = reconciler.SetCondition(pr.Status.Conditions, conditionType, status, reason, message, pr.Generation, clock, logger) +} diff --git a/k8s-operator/reconciler/proxyclass/proxyclass.go b/k8s-operator/reconciler/proxyclass/proxyclass.go index 2ea625500..97268223b 100644 --- a/k8s-operator/reconciler/proxyclass/proxyclass.go +++ b/k8s-operator/reconciler/proxyclass/proxyclass.go @@ -34,7 +34,6 @@ "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler" "tailscale.com/kube/kubetypes" @@ -167,9 +166,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res if errs := r.validate(ctx, pc, logger); errs != nil { msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error()) r.recorder.Event(pc, corev1.EventTypeWarning, ReasonProxyClassInvalid, msg) - tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, ReasonProxyClassInvalid, msg, pc.Generation, r.clock, logger) + reconciler.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, ReasonProxyClassInvalid, msg, pc.Generation, r.clock, logger) } else { - tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, ReasonProxyClassValid, ReasonProxyClassValid, pc.Generation, r.clock, logger) + reconciler.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, ReasonProxyClassValid, ReasonProxyClassValid, pc.Generation, r.clock, logger) } if !apiequality.Semantic.DeepEqual(oldPCStatus, &pc.Status) { if err := r.Client.Status().Update(ctx, pc); err != nil { @@ -480,7 +479,7 @@ func getPortsForProxyClasses(ctx context.Context, c client.Client) (map[string]t portRanges := make(map[string]tsapi.PortRanges) for _, i := range pcs.Items { - if !tsoperator.ProxyClassIsReady(&i) { + if !reconciler.ProxyClassIsReady(&i) { continue } if se := i.Spec.StaticEndpoints; se != nil && se.NodePort != nil { diff --git a/k8s-operator/reconciler/reconciler.go b/k8s-operator/reconciler/reconciler.go index 66339fc96..d50f4c198 100644 --- a/k8s-operator/reconciler/reconciler.go +++ b/k8s-operator/reconciler/reconciler.go @@ -9,20 +9,34 @@ import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" + "net/netip" + "regexp" "slices" "strings" "sync" + "time" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" + "tailscale.com/net/dns/resolvconffile" + "tailscale.com/tailcfg" + "tailscale.com/tstime" "tailscale.com/util/clientmetric" + "tailscale.com/util/dnsname" "tailscale.com/util/set" ) @@ -48,6 +62,52 @@ optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" ) +// Annotations that users set on Services and Ingresses to configure how the operator exposes them. They are read by +// several reconcilers — a Service annotated for egress is acted on by the egress reconcilers, and also by dnsrecords +// when it needs the tailnet target — so they live here rather than in any one reconciler's package. +const ( + // AnnotationTailnetTargetIP is the IP of the tailnet node that an egress Service forwards traffic to. Mutually + // exclusive with AnnotationTailnetTargetFQDN. + AnnotationTailnetTargetIP = "tailscale.com/tailnet-ip" + + // AnnotationTailnetTargetFQDN is the MagicDNS name of the tailnet node that an egress Service forwards traffic + // to. Mutually exclusive with AnnotationTailnetTargetIP. + AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" + + // AnnotationProxyGroup names the ProxyGroup whose proxies should expose an egress Service. + AnnotationProxyGroup = "tailscale.com/proxy-group" + + // AnnotationHostname overrides the tailnet hostname the operator would otherwise derive from a resource's + // namespace and name. + AnnotationHostname = "tailscale.com/hostname" + + // AnnotationTags is a comma-separated list of ACL tags to apply to the tailnet device created for a resource. + AnnotationTags = "tailscale.com/tags" +) + +// Constants used when determining the cluster's DNS domain; see ClusterDomain. +const ( + resolvConfPath = "/etc/resolv.conf" + + // DefaultClusterDomain is the cluster domain assumed when it can't be determined from the resolver config. The + // vast majority of clusters use it. + DefaultClusterDomain = "cluster.local" +) + +// validMagicDNSName matches a tailnet MagicDNS name, e.g. foo.tail-scale.ts.net. +var validMagicDNSName = regexp.MustCompile(`^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+\.ts\.net\.?$`) + +// Reconciler is a CRD reconciler that knows how to register itself with a controller manager, including the watches +// and field indexes it depends on. Every reconciler under this package implements it, so the operator can construct +// them into a slice and register them in a loop rather than wiring each one up by hand. +type Reconciler interface { + reconcile.Reconciler + + // Register sets the reconciler up on mgr. Implementations own their own watches and field indexes, so that a + // caller cannot forget to install something the reconciler needs in order to be triggered. + Register(mgr manager.Manager) error +} + // IsOptimisticLockError reports whether err was caused by an optimistic locking conflict, i.e. the object being // updated was modified after the reconciler read it. Such an error is transient: the reconciler should requeue and // retry against a freshly read object rather than surface the error. @@ -301,3 +361,302 @@ func GetSingleObject[T any, O PtrObject[T]](ctx context.Context, c client.Client } return ret, nil } + +// IsMagicDNSName reports whether name looks like a tailnet MagicDNS name. +func IsMagicDNSName(name string) bool { + return validMagicDNSName.MatchString(name) +} + +// NameForService returns the tailnet hostname to use for svc: the AnnotationHostname override if set, otherwise +// -. +func NameForService(svc *corev1.Service) string { + if h, ok := svc.Annotations[AnnotationHostname]; ok { + return h + } + return svc.Namespace + "-" + svc.Name +} + +// TagViolations returns a human-readable description of every invalid ACL tag in obj's AnnotationTags, or nil if the +// annotation is absent or every tag is valid. +func TagViolations(obj client.Object) []string { + var violations []string + if obj == nil { + return nil + } + tags, ok := obj.GetAnnotations()[AnnotationTags] + if !ok { + return nil + } + + for tag := range strings.SplitSeq(tags, ",") { + tag = strings.TrimSpace(tag) + if err := tailcfg.CheckTag(tag); err != nil { + violations = append(violations, fmt.Sprintf("invalid tag %q: %v", tag, err)) + } + } + return violations +} + +// ValidateService returns a human-readable description of every way in which svc is not a valid Service for the +// operator to act on, or an empty slice if it is valid. Callers append their own resource-specific violations; the +// checks here are the ones common to every Service the operator exposes. +func ValidateService(svc *corev1.Service) []string { + violations := make([]string, 0) + if svc.Spec.ClusterIP == "None" { + violations = append(violations, "headless Services are not supported.") + } + if svc.Annotations[AnnotationTailnetTargetFQDN] != "" && svc.Annotations[AnnotationTailnetTargetIP] != "" { + violations = append(violations, fmt.Sprintf("only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN)) + } + if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" { + if !IsMagicDNSName(fqdn) { + violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn)) + } + } + if ipStr := svc.Annotations[AnnotationTailnetTargetIP]; ipStr != "" { + ip, err := netip.ParseAddr(ipStr) + if err != nil { + violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q could not be parsed as a valid IP Address, error: %s", AnnotationTailnetTargetIP, ipStr, err)) + } else if !ip.IsValid() { + violations = append(violations, fmt.Sprintf("parsed IP address in annotation %s: %q is not valid", AnnotationTailnetTargetIP, ipStr)) + } + } + + svcName := NameForService(svc) + if err := dnsname.ValidLabel(svcName); err != nil { + if _, ok := svc.Annotations[AnnotationHostname]; ok { + violations = append(violations, fmt.Sprintf("invalid Tailscale hostname specified %q: %s", svcName, err)) + } else { + violations = append(violations, fmt.Sprintf("invalid Tailscale hostname %q, use %q annotation to override: %s", svcName, AnnotationHostname, err)) + } + } + violations = append(violations, TagViolations(svc)...) + return violations +} + +// ClusterDomainOption configures ClusterDomain. +type ClusterDomainOption func(*clusterDomainOpts) + +type clusterDomainOpts struct { + resolvConfPath string +} + +// WithResolvConfPath overrides the resolver config that ClusterDomain parses. Only tests should need it: in the +// operator it is the resolver config of the Pod the operator itself runs in. +func WithResolvConfPath(path string) ClusterDomainOption { + return func(o *clusterDomainOpts) { o.resolvConfPath = path } +} + +// ClusterDomain returns the cluster's DNS domain, determined by parsing the resolver config of the Pod the operator +// runs in. It falls back to DefaultClusterDomain when the config is missing or doesn't have the expected shape, since +// erroring out would be worse than assuming the overwhelmingly common value. +func ClusterDomain(namespace string, logger *zap.SugaredLogger, opts ...ClusterDomainOption) string { + o := clusterDomainOpts{resolvConfPath: resolvConfPath} + for _, opt := range opts { + opt(&o) + } + + logger.Infof("attempting to retrieve cluster domain..") + conf, err := resolvconffile.ParseFile(o.resolvConfPath) + if err != nil { + logger.Warnf("error parsing %s to determine cluster domain, defaulting to %q.", o.resolvConfPath, DefaultClusterDomain) + return DefaultClusterDomain + } + return clusterDomainFromResolverConf(conf, namespace, logger) +} + +// clusterDomainFromResolverConf attempts to retrieve cluster domain from the provided resolver config. +// It expects the first three search domains in the resolver config to be ['.svc., svc., , ...] +// If the first three domains match the expected structure, it returns the third. +// If the domains don't match the expected structure or an error is encountered, it defaults to 'cluster.local' domain. +func clusterDomainFromResolverConf(conf *resolvconffile.Config, namespace string, logger *zap.SugaredLogger) string { + if len(conf.SearchDomains) < 3 { + logger.Warnf(" resolver config contains only %d search domains, at least three expected.\nDefaulting cluster domain to 'cluster.local'.", len(conf.SearchDomains)) + return DefaultClusterDomain + } + first := conf.SearchDomains[0] + if !strings.HasPrefix(string(first), namespace+".svc") { + logger.Warnf("first search domain in resolver config is %s; expected %s.\nDefaulting cluster domain to 'cluster.local'.", first, namespace+".svc.") + return DefaultClusterDomain + } + second := conf.SearchDomains[1] + if !strings.HasPrefix(string(second), "svc") { + logger.Warnf("second search domain in resolver config is %s; expected 'svc.'.\nDefaulting cluster domain to 'cluster.local'.", second) + return DefaultClusterDomain + } + // Trim the trailing dot for backwards compatibility purposes as the + // cluster domain was previously hardcoded to 'cluster.local' without a + // trailing dot. + probablyClusterDomain := strings.TrimPrefix(second.WithoutTrailingDot(), "svc.") + third := conf.SearchDomains[2] + if !strings.EqualFold(third.WithoutTrailingDot(), probablyClusterDomain) { + logger.Warnf("expected resolver config to contain serch domains .svc., svc., ; got %s %s %s\n. Defaulting cluster domain to 'cluster.local'.", first, second, third) + return DefaultClusterDomain + } + logger.Infof("Cluster domain %q extracted from resolver config", probablyClusterDomain) + return probablyClusterDomain +} + +// TruncateLabelValue truncates a Kubernetes label value to fit within the +// 63-character limit. If the value exceeds the limit, it is truncated and a +// short hash suffix is appended to preserve uniqueness. +func TruncateLabelValue(val string) string { + const maxLen = 63 + if len(val) <= maxLen { + return val + } + hash := sha256.Sum256([]byte(val)) + suffix := hex.EncodeToString(hash[:4]) // 8 hex chars + truncated := val[:maxLen-len(suffix)-1] + return truncated + "-" + suffix +} + +// SetCondition ensures conds has a condition of the given type with the supplied status, reason, message and observed +// generation, and returns the updated slice. LastTransitionTime is only advanced when the status actually changes, so +// callers can call it on every reconcile without churning the resource; a transition is logged when it happens. +// +// It operates on the condition slice rather than the owning object because the Tailscale CRD types expose +// Status.Conditions as a plain field with no accessor to constrain a type parameter on. Each reconciler wraps this in +// a small helper for its own CRD, e.g. +// +// func setStatus(pr *tsapi.PeerRelay, ct tsapi.ConditionType, st metav1.ConditionStatus, reason, msg string, clock tstime.Clock, lg *zap.SugaredLogger) { +// pr.Status.Conditions = reconciler.SetCondition(pr.Status.Conditions, ct, st, reason, msg, pr.Generation, clock, lg) +// } +func SetCondition(conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []metav1.Condition { + newCondition := metav1.Condition{ + Type: string(conditionType), + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: gen, + } + + newCondition.LastTransitionTime = metav1.NewTime(clock.Now().Truncate(time.Second)) + + idx := slices.IndexFunc(conds, func(cond metav1.Condition) bool { + return cond.Type == string(conditionType) + }) + if idx == -1 { + return append(conds, newCondition) + } + + cond := conds[idx] // update the existing condition + + // If this update doesn't contain a state transition, don't update last + // transition time. + if cond.Status == status { + newCondition.LastTransitionTime = cond.LastTransitionTime + } else { + logger.Infof("Status change for condition %s from %s to %s", conditionType, cond.Status, status) + } + conds[idx] = newCondition + return conds +} + +// Condition returns the condition of the given type from conds, or nil if it isn't present. +func Condition(conds []metav1.Condition, conditionType tsapi.ConditionType) *metav1.Condition { + idx := slices.IndexFunc(conds, func(cond metav1.Condition) bool { + return cond.Type == string(conditionType) + }) + if idx == -1 { + return nil + } + return &conds[idx] +} + +// RemoveCondition removes the condition of the given type from conds if present, returning the updated slice. +func RemoveCondition(conds []metav1.Condition, conditionType tsapi.ConditionType) []metav1.Condition { + return slices.DeleteFunc(conds, func(cond metav1.Condition) bool { + return cond.Type == string(conditionType) + }) +} + +// The helpers below wrap SetCondition and friends for the CRDs whose reconcilers still live in cmd/k8s-operator. As +// each of those moves into its own package under this one it should grow a package-local wrapper instead, at which +// point the corresponding helper here can go. + +// SetConnectorCondition ensures that Connector status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes. +func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { + cn.Status.Conditions = SetCondition(cn.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) +} + +// SetProxyClassCondition ensures that ProxyClass status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes. +func SetProxyClassCondition(pc *tsapi.ProxyClass, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { + pc.Status.Conditions = SetCondition(pc.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) +} + +// SetDNSConfigCondition ensures that DNSConfig status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes +func SetDNSConfigCondition(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { + dnsCfg.Status.Conditions = SetCondition(dnsCfg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) +} + +// SetServiceCondition ensures that Service status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes. Services carry no meaningful generation for the operator's purposes, so ObservedGeneration is always 0. +func SetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) { + svc.Status.Conditions = SetCondition(svc.Status.Conditions, conditionType, status, reason, message, 0, clock, logger) +} + +// GetServiceCondition returns Service condition with the specified type, if it exists on the Service. +func GetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) *metav1.Condition { + return Condition(svc.Status.Conditions, conditionType) +} + +// RemoveServiceCondition will remove condition of the given type if it exists. +func RemoveServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) { + svc.Status.Conditions = RemoveCondition(svc.Status.Conditions, conditionType) +} + +// SetProxyGroupCondition ensures that ProxyGroup status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes. +func SetProxyGroupCondition(pg *tsapi.ProxyGroup, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { + pg.Status.Conditions = SetCondition(pg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) +} + +// ProxyClassIsReady reports whether pc has been validated for its current generation. +func ProxyClassIsReady(pc *tsapi.ProxyClass) bool { + cond := Condition(pc.Status.Conditions, tsapi.ProxyClassReady) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pc.Generation +} + +// ProxyGroupAvailable reports whether at least one of pg's proxies is running and ready to serve traffic. +func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool { + cond := Condition(pg.Status.Conditions, tsapi.ProxyGroupAvailable) + return cond != nil && cond.Status == metav1.ConditionTrue +} + +// ProxyGroupReplicas returns the number of replicas the ProxyGroup asks for, defaulting to two when unset. +func ProxyGroupReplicas(pg *tsapi.ProxyGroup) int32 { + if pg.Spec.Replicas != nil { + return *pg.Spec.Replicas + } + + return 2 +} + +// KubeAPIServerProxyValid reports whether pg's kube-apiserver proxy config has been validated for its current +// generation, and whether the condition is present at all. +func KubeAPIServerProxyValid(pg *tsapi.ProxyGroup) (valid bool, set bool) { + cond := Condition(pg.Status.Conditions, tsapi.KubeAPIServerProxyValid) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation, cond != nil +} + +// KubeAPIServerProxyConfigured reports whether pg's kube-apiserver proxy has been configured for its current +// generation. +func KubeAPIServerProxyConfigured(pg *tsapi.ProxyGroup) bool { + cond := Condition(pg.Status.Conditions, tsapi.KubeAPIServerProxyConfigured) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation +} + +// SvcIsReady reports whether the proxy exposing svc is ready. +func SvcIsReady(svc *corev1.Service) bool { + cond := Condition(svc.Status.Conditions, tsapi.ProxyReady) + return cond != nil && cond.Status == metav1.ConditionTrue +} diff --git a/k8s-operator/reconciler/reconciler_test.go b/k8s-operator/reconciler/reconciler_test.go index 1e6f54bdf..58d6eeb39 100644 --- a/k8s-operator/reconciler/reconciler_test.go +++ b/k8s-operator/reconciler/reconciler_test.go @@ -9,9 +9,16 @@ "errors" "fmt" "maps" + "os" + "path/filepath" "slices" + "strings" "testing" + "time" + "go.uber.org/zap" + + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -20,7 +27,9 @@ "sigs.k8s.io/controller-runtime/pkg/client/fake" ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler" + "tailscale.com/tstest" "tailscale.com/util/clientmetric" ) @@ -294,3 +303,233 @@ func TestEnsureAndClearFinalizer(t *testing.T) { } }) } + +func TestClusterDomain(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + // resolvConf is the file ClusterDomain parses. Empty means don't create the file at all, exercising the + // unreadable-config path. + resolvConf string + namespace string + want string + }{ + "custom-domain": { + resolvConf: "search foo.svc.department.org.io svc.department.org.io department.org.io\nnameserver 10.96.0.10\n", + namespace: "foo", + want: "department.org.io", + }, + "default-domain": { + resolvConf: "search foo.svc.cluster.local svc.cluster.local cluster.local\nnameserver 10.96.0.10\n", + namespace: "foo", + want: "cluster.local", + }, + // Everything below should fall back to the default rather than error. + "only-two-search-domains": { + resolvConf: "search svc.department.org.io department.org.io\n", + namespace: "foo", + want: "cluster.local", + }, + "first-search-domain-mismatch": { + resolvConf: "search foo.bar.department.org.io svc.department.org.io some.other.fqdn\n", + namespace: "foo", + want: "cluster.local", + }, + "second-search-domain-mismatch": { + resolvConf: "search foo.svc.department.org.io foo.department.org.io some.other.fqdn\n", + namespace: "foo", + want: "cluster.local", + }, + "third-search-domain-mismatch": { + resolvConf: "search foo.svc.department.org.io svc.department.org.io some.other.fqdn\n", + namespace: "foo", + want: "cluster.local", + }, + // The domain here is deliberately not cluster.local: if the namespace check were skipped, this config + // would parse cleanly and yield department.org.io, so the expected fallback distinguishes the two. + "namespace-mismatch": { + resolvConf: "search bar.svc.department.org.io svc.department.org.io department.org.io\n", + namespace: "foo", + want: "cluster.local", + }, + "no-search-domains": { + resolvConf: "nameserver 10.96.0.10\n", + namespace: "foo", + want: "cluster.local", + }, + "missing-file": { + namespace: "foo", + want: "cluster.local", + }, + } + + logger := zap.Must(zap.NewDevelopment()).Sugar() + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "resolv.conf") + if tc.resolvConf != "" { + if err := os.WriteFile(path, []byte(tc.resolvConf), 0600); err != nil { + t.Fatalf("writing resolv.conf: %v", err) + } + } + + if got := reconciler.ClusterDomain(tc.namespace, logger, reconciler.WithResolvConfPath(path)); got != tc.want { + t.Errorf("ClusterDomain() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestTruncateLabelValue(t *testing.T) { + tests := []struct { + name string + input string + want string // empty means expect input unchanged + }{ + { + name: "short-value-unchanged", + input: "my-service", + }, + { + name: "exactly-63-chars-unchanged", + input: strings.Repeat("a", 63), + }, + { + name: "64-chars-gets-truncated", + input: strings.Repeat("a", 64), + }, + { + name: "very-long-value-gets-truncated", + input: "tailscale-nginx-clickhouse-o11y-server-https-with-extra-long-suffix-that-exceeds-limit", + }, + { + name: "253-chars-max-k8s-resource-name", + input: strings.Repeat("x", 253), + }, + { + name: "empty-string-unchanged", + input: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := reconciler.TruncateLabelValue(tt.input) + if len(got) > 63 { + t.Errorf("reconciler.TruncateLabelValue(%q) = %q (len %d), exceeds 63 chars", tt.input, got, len(got)) + } + if len(tt.input) <= 63 && got != tt.input { + t.Errorf("reconciler.TruncateLabelValue(%q) = %q, want unchanged input", tt.input, got) + } + if len(tt.input) > 63 && got == tt.input { + t.Errorf("reconciler.TruncateLabelValue(%q) was not truncated", tt.input) + } + }) + } +} + +func TestTruncateLabelValueDeterministic(t *testing.T) { + input := strings.Repeat("a", 100) + first := reconciler.TruncateLabelValue(input) + for range 10 { + got := reconciler.TruncateLabelValue(input) + if got != first { + t.Fatalf("non-deterministic: got %q, want %q", got, first) + } + } +} + +func TestTruncateLabelValueUniqueness(t *testing.T) { + // Two inputs sharing a long prefix but differing at the end should produce different outputs. + a := strings.Repeat("a", 100) + "-one" + b := strings.Repeat("a", 100) + "-two" + if reconciler.TruncateLabelValue(a) == reconciler.TruncateLabelValue(b) { + t.Errorf("collision: %q and %q produce the same truncated label", a, b) + } +} + +func TestSetConnectorCondition(t *testing.T) { + cn := tsapi.Connector{} + clock := tstest.NewClock(tstest.ClockOpts{}) + fakeNow := metav1.NewTime(clock.Now().Truncate(time.Second)) + fakePast := metav1.NewTime(clock.Now().Truncate(time.Second).Add(-5 * time.Minute)) + zl, err := zap.NewDevelopment() + assert.Nil(t, err) + + // Set up a new condition + reconciler.SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "someReason", "someMsg", 1, clock, zl.Sugar()) + assert.Equal(t, cn, tsapi.Connector{ + Status: tsapi.ConnectorStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ConnectorReady), + Status: metav1.ConditionTrue, + Reason: "someReason", + Message: "someMsg", + ObservedGeneration: 1, + LastTransitionTime: fakeNow, + }, + }, + }, + }) + + // Modify status of an existing condition + cn.Status = tsapi.ConnectorStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ConnectorReady), + Status: metav1.ConditionFalse, + Reason: "someReason", + Message: "someMsg", + ObservedGeneration: 1, + LastTransitionTime: fakePast, + }, + }, + } + reconciler.SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar()) + assert.Equal(t, cn, tsapi.Connector{ + Status: tsapi.ConnectorStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ConnectorReady), + Status: metav1.ConditionTrue, + Reason: "anotherReason", + Message: "anotherMsg", + ObservedGeneration: 2, + LastTransitionTime: fakeNow, + }, + }, + }, + }) + + // Don't modify last transition time if status hasn't changed + cn.Status = tsapi.ConnectorStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ConnectorReady), + Status: metav1.ConditionTrue, + Reason: "someReason", + Message: "someMsg", + ObservedGeneration: 1, + LastTransitionTime: fakePast, + }, + }, + } + reconciler.SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar()) + assert.Equal(t, cn, tsapi.Connector{ + Status: tsapi.ConnectorStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ConnectorReady), + Status: metav1.ConditionTrue, + Reason: "anotherReason", + Message: "anotherMsg", + ObservedGeneration: 2, + LastTransitionTime: fakePast, + }, + }, + }, + }) +} diff --git a/k8s-operator/reconciler/reconcilertest/conditions.go b/k8s-operator/reconciler/reconcilertest/conditions.go index 0b88b9ceb..8eadf62ba 100644 --- a/k8s-operator/reconciler/reconcilertest/conditions.go +++ b/k8s-operator/reconciler/reconcilertest/conditions.go @@ -7,11 +7,13 @@ import ( "testing" + "time" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstime" ) // The condition helpers below take a []metav1.Condition rather than the owning object. Every Tailscale CRD stores its @@ -57,3 +59,9 @@ func ExpectConditionStatus(t *testing.T, conds []metav1.Condition, conditionType conditionType, got.Status, got.Reason, status, reason) } } + +// ConditionTime returns the LastTransitionTime a reconciler using clock would stamp on a condition. The operator +// truncates to the second, so a test building an expected condition has to do the same. +func ConditionTime(clock tstime.Clock) metav1.Time { + return metav1.NewTime(clock.Now().Truncate(time.Second)) +} diff --git a/k8s-operator/reconciler/reconcilertest/reconcilertest.go b/k8s-operator/reconciler/reconcilertest/reconcilertest.go index 5c9c5ff85..07da4473e 100644 --- a/k8s-operator/reconciler/reconcilertest/reconcilertest.go +++ b/k8s-operator/reconciler/reconcilertest/reconcilertest.go @@ -24,6 +24,7 @@ "time" "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -78,6 +79,14 @@ func MustCreate(t *testing.T, c client.Client, obj client.Object) { } } +// MustCreateAll creates each of objs in order. +func MustCreateAll(t *testing.T, c client.Client, objs ...client.Object) { + t.Helper() + for _, obj := range objs { + MustCreate(t, c, obj) + } +} + // MustGet reads ns/name into obj, failing the test if it is absent. Use ExpectMissing to assert the opposite. func MustGet(t *testing.T, c client.Client, ns, name string, obj client.Object) { t.Helper() @@ -176,6 +185,27 @@ func ExpectReconciled(t *testing.T, r reconcile.Reconciler, ns, name string) { } } +// ExpectRequeue runs a single reconcile for ns/name and asserts it succeeded but asked to be retried later, i.e. the +// reconciler is waiting on state it doesn't control. +func ExpectRequeue(t *testing.T, r reconcile.Reconciler, ns, name string) { + t.Helper() + res, err := r.Reconcile(t.Context(), request(ns, name)) + if err != nil { + t.Fatalf("Reconcile: unexpected error: %v", err) + } + if res.RequeueAfter == 0 { + t.Fatalf("expected timed requeue, got success") + } +} + +// ExpectReconcileError runs a single reconcile for ns/name and asserts it returned an error. +func ExpectReconcileError(t *testing.T, r reconcile.Reconciler, ns, name string) { + t.Helper() + if _, err := r.Reconcile(t.Context(), request(ns, name)); err == nil { + t.Error("Reconcile: expected error but did not get one") + } +} + func request(ns, name string) reconcile.Request { return reconcile.Request{NamespacedName: types.NamespacedName{Namespace: ns, Name: name}} } @@ -200,3 +230,20 @@ func ExpectEvents(t *testing.T, rec *record.FakeRecorder, want []string) { } } } + +// RemoveTargetPorts is an ExpectEqual modifier that blanks the TargetPort of every port on a Service. Reconcilers that +// allocate a target port pick it at random, so a test can assert on the rest of the Service without pinning it. +func RemoveTargetPorts(svc *corev1.Service) { + ports := make([]corev1.ServicePort, 0, len(svc.Spec.Ports)) + for _, p := range svc.Spec.Ports { + ports = append(ports, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port, Name: p.Name}) + } + svc.Spec.Ports = ports +} + +// RemoveClusterIPs is an ExpectEqual modifier that blanks a Service's ClusterIPs, which are assigned by the cluster +// rather than by the reconciler under test. +func RemoveClusterIPs(svc *corev1.Service) { + svc.Spec.ClusterIP = "" + svc.Spec.ClusterIPs = nil +} diff --git a/k8s-operator/reconciler/recorder/recorder.go b/k8s-operator/reconciler/recorder/recorder.go index 8f5e43f40..3a114cfbb 100644 --- a/k8s-operator/reconciler/recorder/recorder.go +++ b/k8s-operator/reconciler/recorder/recorder.go @@ -33,7 +33,6 @@ "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/reconciler/tailscaled" @@ -158,7 +157,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco oldTSRStatus := tsr.Status.DeepCopy() setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger) + tsr.Status.Conditions = reconciler.SetCondition(tsr.Status.Conditions, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger) if !apiequality.Semantic.DeepEqual(oldTSRStatus, &tsr.Status) { // An error encountered here should get returned by the Reconcile function. if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil { diff --git a/k8s-operator/reconciler/tailnet/tailnet.go b/k8s-operator/reconciler/tailnet/tailnet.go index 31e2411dc..bd6576adc 100644 --- a/k8s-operator/reconciler/tailnet/tailnet.go +++ b/k8s-operator/reconciler/tailnet/tailnet.go @@ -28,7 +28,6 @@ "tailscale.com/client/tailscale/v2" "tailscale.com/ipn" - operatorutils "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/reconciler" "tailscale.com/k8s-operator/tsclient" @@ -165,7 +164,7 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet) // The referenced Secret does not exist within the tailscale namespace, so we'll mark the Tailnet as not ready // for use. if apierrors.IsNotFound(err) { - operatorutils.SetTailnetCondition( + setCondition( tailnet, tsapi.TailnetReady, metav1.ConditionFalse, @@ -215,7 +214,7 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet) return reconcile.Result{RequeueAfter: time.Minute / 2}, nil } - operatorutils.SetTailnetCondition( + setCondition( tailnet, tsapi.TailnetReady, metav1.ConditionTrue, @@ -334,7 +333,7 @@ func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient tsclient.Cl } if errs != nil { - operatorutils.SetTailnetCondition( + setCondition( tailnet, tsapi.TailnetReady, metav1.ConditionFalse, @@ -366,7 +365,7 @@ func (r *Reconciler) ensureSecret(tailnet *tsapi.Tailnet, secret *corev1.Secret) return true } - operatorutils.SetTailnetCondition( + setCondition( tailnet, tsapi.TailnetReady, metav1.ConditionFalse, @@ -378,3 +377,9 @@ func (r *Reconciler) ensureSecret(tailnet *tsapi.Tailnet, secret *corev1.Secret) return false } + +// setCondition sets a condition on tn's status. ObservedGeneration is always the Tailnet's own generation, so callers +// don't pass it. +func setCondition(tn *tsapi.Tailnet, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) { + tn.Status.Conditions = reconciler.SetCondition(tn.Status.Conditions, conditionType, status, reason, message, tn.Generation, clock, logger) +} diff --git a/k8s-operator/utils.go b/k8s-operator/utils.go index 8d1391383..4a36214e3 100644 --- a/k8s-operator/utils.go +++ b/k8s-operator/utils.go @@ -7,8 +7,6 @@ package kube import ( - "crypto/sha256" - "encoding/hex" "fmt" "net/netip" "strconv" @@ -94,17 +92,3 @@ func ResolveViaDomain(name string) (netip.Addr, bool) { out, _ := tsaddr.MapVia(uint32(prefix), netip.PrefixFrom(ip4, ip4.BitLen())) return out.Addr(), true } - -// TruncateLabelValue truncates a Kubernetes label value to fit within the -// 63-character limit. If the value exceeds the limit, it is truncated and a -// short hash suffix is appended to preserve uniqueness. -func TruncateLabelValue(val string) string { - const maxLen = 63 - if len(val) <= maxLen { - return val - } - hash := sha256.Sum256([]byte(val)) - suffix := hex.EncodeToString(hash[:4]) // 8 hex chars - truncated := val[:maxLen-len(suffix)-1] - return truncated + "-" + suffix -} diff --git a/k8s-operator/utils_test.go b/k8s-operator/utils_test.go index d46f5e64f..5388cd026 100644 --- a/k8s-operator/utils_test.go +++ b/k8s-operator/utils_test.go @@ -3,76 +3,88 @@ //go:build !plan9 -package kube +package kube_test import ( - "strings" + "fmt" + "net/netip" "testing" + + kube "tailscale.com/k8s-operator" + "tailscale.com/tailcfg" ) -func TestTruncateLabelValue(t *testing.T) { - tests := []struct { - name string - input string - want string // empty means expect input unchanged - }{ - { - name: "short-value-unchanged", - input: "my-service", - }, - { - name: "exactly-63-chars-unchanged", - input: strings.Repeat("a", 63), - }, - { - name: "64-chars-gets-truncated", - input: strings.Repeat("a", 64), - }, - { - name: "very-long-value-gets-truncated", - input: "tailscale-nginx-clickhouse-o11y-server-https-with-extra-long-suffix-that-exceeds-limit", - }, - { - name: "253-chars-max-k8s-resource-name", - input: strings.Repeat("x", 253), - }, - { - name: "empty-string-unchanged", - input: "", - }, +// TestTailscaledConfigFileNameRoundTrip pins the config file naming contract. The operator writes these names into a +// Secret and containerboot parses them back out, so the two functions must stay inverses of each other and the format +// must not drift. +func TestTailscaledConfigFileNameRoundTrip(t *testing.T) { + t.Parallel() + + for _, capVer := range []tailcfg.CapabilityVersion{0, 1, 95, 106, 32767} { + name := kube.TailscaledConfigFileName(capVer) + if want := fmt.Sprintf("cap-%d.hujson", capVer); name != want { + t.Errorf("TailscaledConfigFileName(%d) = %q, want %q", capVer, name, want) + } + + got, err := kube.CapVerFromFileName(name) + if err != nil { + t.Errorf("CapVerFromFileName(%q): %v", name, err) + continue + } + if got != capVer { + t.Errorf("round trip of %d gave %d", capVer, got) + } } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := TruncateLabelValue(tt.input) - if len(got) > 63 { - t.Errorf("TruncateLabelValue(%q) = %q (len %d), exceeds 63 chars", tt.input, got, len(got)) + + // Pre-config-file proxies wrote a file simply named "tailscaled"; it maps to capability version 0. + got, err := kube.CapVerFromFileName("tailscaled") + if err != nil { + t.Errorf(`CapVerFromFileName("tailscaled"): %v`, err) + } + if got != 0 { + t.Errorf(`CapVerFromFileName("tailscaled") = %d, want 0`, got) + } + + if _, err := kube.CapVerFromFileName("not-a-config-file"); err == nil { + t.Error("CapVerFromFileName(\"not-a-config-file\") succeeded, want error") + } +} + +func TestResolveViaDomain(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + name string + want string // empty means expect ok == false + }{ + "bare": {name: "10-1-2-3-via-7", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"}, + "trailing-dot": {name: "10-1-2-3-via-7.", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"}, + "ts.net-domain": {name: "10-1-2-3-via-7.foo.ts.net.", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"}, + "tailscale.net-domain": {name: "10-1-2-3-via-7.foo.tailscale.net", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"}, + "hex-site-id": {name: "10-1-2-3-via-0x7", want: "fd7a:115c:a1e0:b1a:0:7:a01:203"}, + "too-short": {name: "0-0-0-via"}, + "no-via": {name: "10-1-2-3-7.foo.ts.net"}, + "foreign-domain": {name: "10-1-2-3-via-7.example.com"}, + "not-an-ipv4": {name: "10-1-2-via-7"}, + "site-id-not-a-number": {name: "10-1-2-3-via-abc"}, + } + + for tn, tc := range tests { + t.Run(tn, func(t *testing.T) { + t.Parallel() + got, ok := kube.ResolveViaDomain(tc.name) + if tc.want == "" { + if ok { + t.Fatalf("ResolveViaDomain(%q) = %v, true; want ok == false", tc.name, got) + } + return } - if len(tt.input) <= 63 && got != tt.input { - t.Errorf("TruncateLabelValue(%q) = %q, want unchanged input", tt.input, got) + if !ok { + t.Fatalf("ResolveViaDomain(%q) returned ok == false, want %s", tc.name, tc.want) } - if len(tt.input) > 63 && got == tt.input { - t.Errorf("TruncateLabelValue(%q) was not truncated", tt.input) + if got != netip.MustParseAddr(tc.want) { + t.Errorf("ResolveViaDomain(%q) = %v, want %v", tc.name, got, tc.want) } }) } } - -func TestTruncateLabelValueDeterministic(t *testing.T) { - input := strings.Repeat("a", 100) - first := TruncateLabelValue(input) - for range 10 { - got := TruncateLabelValue(input) - if got != first { - t.Fatalf("non-deterministic: got %q, want %q", got, first) - } - } -} - -func TestTruncateLabelValueUniqueness(t *testing.T) { - // Two inputs sharing a long prefix but differing at the end should produce different outputs. - a := strings.Repeat("a", 100) + "-one" - b := strings.Repeat("a", 100) + "-two" - if TruncateLabelValue(a) == TruncateLabelValue(b) { - t.Errorf("collision: %q and %q produce the same truncated label", a, b) - } -}