cmd/k8s-operator: give kube-apiserver ProxyGroups the configured log level (#21888)

The operator set each kube-apiserver ProxyGroup's LogLevel from its own
logger's Level(). Unless TS_NO_LOGS_NO_SUPPORT is set, that logger is
teed into the log upload core at debug, so Level() is always debug and
every k8s-proxy logged at debug whatever OPERATOR_LOGGING said. In our
fleet that is millions of debug lines an hour from the proxies.

Read the level before adding the tee and hand it to the ProxyGroup
reconciler. Log uploads are unchanged.

Fixes #21887

Change-Id: I3dd791549c60ba2a4f3c2651da6716623da571a6

Signed-off-by: sugarraysam <sbd@fireworks.ai>
Co-authored-by: Cursor <cursoragent@cursor.com>

cmd/k8s-operator/operator.go

@@ -150,6 +150,9 @@ func main() {
 		}()
 	}
 
+	// Read before the log upload tee below, which lowers the logger's level to debug.
+	logLevel := zlog.Level()
+
 	// Operator log uploads can be opted-out using the "TS_NO_LOGS_NO_SUPPORT" environment variable.
 	if !envknob.NoLogsNoSupport() {
 		zlog = zlog.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
@@ -159,6 +162,7 @@ func main() {
 
 	runReconcilers(reconcilerOpts{
 		log:                           zlog,
+		logLevel:                      logLevel,
 		tsServer:                      s,
 		tsClient:                      tsc,
 		tailscaleNamespace:            tsNamespace,
@@ -753,6 +757,7 @@ func runReconcilers(opts reconcilerOpts) {
 			tsFirewallMode:    opts.proxyFirewallMode,
 			defaultProxyClass: opts.defaultProxyClass,
 			loginServer:       opts.tsServer.ControlURL,
+			logLevel:          opts.logLevel,
 			reissuer:          tailscaled.NewReissuer(),
 
 			sharedACMEAccountKey: opts.sharedACMEAccountKey,
@@ -769,6 +774,7 @@ func runReconcilers(opts reconcilerOpts) {
 
 type reconcilerOpts struct {
 	log                *zap.SugaredLogger
+	logLevel           zapcore.Level // configured level; log's own level is debug while log uploads are on
 	tsServer           *tsnet.Server
 	tsClient           *tailscale.Client
 	tailscaleNamespace string       // namespace in which operator resources will be deployed

cmd/k8s-operator/proxygroup.go

@@ -18,6 +18,7 @@ import (
 
 	dockerref "github.com/distribution/reference"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 	xslices "golang.org/x/exp/slices"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -93,6 +94,7 @@ type ProxyGroupReconciler struct {
 	tsFirewallMode    string
 	defaultProxyClass string
 	loginServer       string
+	logLevel          zapcore.Level // kube-apiserver proxies log at this level
 
 	reissuer *tailscaled.Reissuer
 
@@ -710,7 +712,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(
 					AuthKey:  authKey,
 					State:    new(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))),
 					App:      new(kubetypes.AppProxyGroupKubeAPIServer),
-					LogLevel: new(logger.Level().String()),
+					LogLevel: new(r.logLevel.String()),
 
 					// Reloadable fields.
 					Hostname: &hostname,

cmd/k8s-operator/proxygroup_test.go

@@ -1566,7 +1566,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) {
 			AuthKey:  new("new-authkey"),
 			State:    new(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))),
 			App:      new(kubetypes.AppProxyGroupKubeAPIServer),
-			LogLevel: new("debug"),
+			LogLevel: new("info"), // not the reconciler logger's debug
 
 			Hostname: new("test-k8s-apiserver-0"),
 			APIServerProxy: &conf.APIServerProxyConfig{