From 9020fdcd8ae91c2fda38af448f2976f2e5b002d1 Mon Sep 17 00:00:00 2001
From: kari-ts <135075563+kari-ts@users.noreply.github.com>
Date: Thu, 24 Sep 2026 10:43:53 -0700
Subject: [PATCH] logtail: stop retrying uploads after logger is disabled
 (#21459)

Logger.SetEnabled(false) prevents new log entries from being buffered, but a batch that has already been drained into uploading can continue retrying indefinitely after logging is disabled.  This was seen on a client where log.tailscale.com was blocked and remote client logging had been disabled.
Here we add a check on the disabled state before upload attempts so that existing failed batches are abandoned after logging is disabled.

Updates tailscale/tailscale#21088

Signed-off-by: kari-ts <kari@tailscale.com>
---
 logtail/logtail.go      |  3 +++
 logtail/logtail_test.go | 49 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/logtail/logtail.go b/logtail/logtail.go
index 14cdf21af..f8e1d103b 100644
--- a/logtail/logtail.go
+++ b/logtail/logtail.go
@@ -548,6 +548,9 @@ func (lg *Logger) uploading(ctx context.Context) {
 		var numFailures int
 		var firstFailure time.Time
 		for len(body) > 0 && ctx.Err() == nil {
+			if logtailDisabled.Load() || lg.disabled.Load() {
+				break
+			}
 			retryAfter, err := lg.upload(ctx, body, origlen)
 			if err != nil {
 				numFailures++
diff --git a/logtail/logtail_test.go b/logtail/logtail_test.go
index ef56f37e2..0c1b62670 100644
--- a/logtail/logtail_test.go
+++ b/logtail/logtail_test.go
@@ -15,6 +15,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"testing/synctest"
 	"time"
@@ -666,6 +667,54 @@ func TestLoggerSetEnabled(t *testing.T) {
 	}
 }
 
+func TestLoggerSetEnabledStopsPendingRetry(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		var calls atomic.Int32
+
+		httpc := &http.Client{
+			Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
+				calls.Add(1)
+				return &http.Response{
+					StatusCode: http.StatusServiceUnavailable,
+					Header: http.Header{
+						"Retry-After": []string{"1"},
+					},
+					Body: io.NopCloser(strings.NewReader("try again")),
+				}, nil
+			}),
+		}
+
+		lg := NewLogger(Config{
+			BaseURL:      "http://logtail.test.invalid",
+			HTTPC:        httpc,
+			Bus:          eventbustest.NewBus(t),
+			FlushDelayFn: func() time.Duration { return 0 },
+		}, t.Logf)
+		defer func() {
+			ctx, cancel := context.WithCancel(context.Background())
+			cancel()
+			lg.Shutdown(ctx)
+		}()
+
+		lg.Logf("hello")
+
+		synctest.Wait()
+
+		if got := calls.Load(); got != 1 {
+			t.Fatalf("upload attempts before disabling = %d; want 1", got)
+		}
+
+		lg.SetEnabled(false)
+
+		time.Sleep(time.Second)
+		synctest.Wait()
+
+		if got := calls.Load(); got != 1 {
+			t.Fatalf("upload retried after SetEnabled(false): got %d attempts, want 1", got)
+		}
+	})
+}
+
 func TestAppendMetadata(t *testing.T) {
 	var lg Logger
 	lg.clock = tstest.NewClock(tstest.ClockOpts{Start: time.Date(2000, 01, 01, 0, 0, 0, 0, time.UTC)})