From c9e36203f4c8f77e06140ee0896900a3fe8cc1ec Mon Sep 17 00:00:00 2001 From: Samuel Blais-Dowdy Date: Tue, 27 Oct 2026 07:42:44 -0700 Subject: [PATCH] cmd/k8s-operator: give kube-apiserver ProxyGroups the configured log level (#21888) The operator set each kube-apiserver ProxyGroup's LogLevel from its own logger's Level(). Unless TS_NO_LOGS_NO_SUPPORT is set, that logger is teed into the log upload core at debug, so Level() is always debug and every k8s-proxy logged at debug whatever OPERATOR_LOGGING said. In our fleet that is millions of debug lines an hour from the proxies. Read the level before adding the tee and hand it to the ProxyGroup reconciler. Log uploads are unchanged. Fixes #21887 Change-Id: I3dd791549c60ba2a4f3c2651da6716623da571a6 Signed-off-by: sugarraysam Co-authored-by: Cursor --- cmd/k8s-operator/operator.go | 6 ++++++ cmd/k8s-operator/proxygroup.go | 4 +++- cmd/k8s-operator/proxygroup_test.go | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 8f621d506..2619d322f 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -150,6 +150,9 @@ func main() { }() } + // Read before the log upload tee below, which lowers the logger's level to debug. + logLevel := zlog.Level() + // Operator log uploads can be opted-out using the "TS_NO_LOGS_NO_SUPPORT" environment variable. if !envknob.NoLogsNoSupport() { zlog = zlog.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { @@ -159,6 +162,7 @@ func main() { runReconcilers(reconcilerOpts{ log: zlog, + logLevel: logLevel, tsServer: s, tsClient: tsc, tailscaleNamespace: tsNamespace, @@ -753,6 +757,7 @@ func runReconcilers(opts reconcilerOpts) { tsFirewallMode: opts.proxyFirewallMode, defaultProxyClass: opts.defaultProxyClass, loginServer: opts.tsServer.ControlURL, + logLevel: opts.logLevel, reissuer: tailscaled.NewReissuer(), sharedACMEAccountKey: opts.sharedACMEAccountKey, @@ -769,6 +774,7 @@ func runReconcilers(opts reconcilerOpts) { type reconcilerOpts struct { log *zap.SugaredLogger + logLevel zapcore.Level // configured level; log's own level is debug while log uploads are on tsServer *tsnet.Server tsClient *tailscale.Client tailscaleNamespace string // namespace in which operator resources will be deployed diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 373f1d607..1c1da1118 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -18,6 +18,7 @@ import ( dockerref "github.com/distribution/reference" "go.uber.org/zap" + "go.uber.org/zap/zapcore" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -93,6 +94,7 @@ type ProxyGroupReconciler struct { tsFirewallMode string defaultProxyClass string loginServer string + logLevel zapcore.Level // kube-apiserver proxies log at this level reissuer *tailscaled.Reissuer @@ -710,7 +712,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated( AuthKey: authKey, State: new(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))), App: new(kubetypes.AppProxyGroupKubeAPIServer), - LogLevel: new(logger.Level().String()), + LogLevel: new(r.logLevel.String()), // Reloadable fields. Hostname: &hostname, diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index fdc57b073..304b7a237 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -1566,7 +1566,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { AuthKey: new("new-authkey"), State: new(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))), App: new(kubetypes.AppProxyGroupKubeAPIServer), - LogLevel: new("debug"), + LogLevel: new("info"), // not the reconciler logger's debug Hostname: new("test-k8s-apiserver-0"), APIServerProxy: &conf.APIServerProxyConfig{