logtail: stop retrying uploads after logger is disabled (#21459)

Logger.SetEnabled(false) prevents new log entries from being buffered, but a batch that has already been drained into uploading can continue retrying indefinitely after logging is disabled.  This was seen on a client where log.tailscale.com was blocked and remote client logging had been disabled.
Here we add a check on the disabled state before upload attempts so that existing failed batches are abandoned after logging is disabled.

Updates tailscale/tailscale#21088

Signed-off-by: kari-ts <kari@tailscale.com>

logtail/logtail.go

@@ -548,6 +548,9 @@ func (lg *Logger) uploading(ctx context.Context) {
 		var numFailures int
 		var firstFailure time.Time
 		for len(body) > 0 && ctx.Err() == nil {
+			if logtailDisabled.Load() || lg.disabled.Load() {
+				break
+			}
 			retryAfter, err := lg.upload(ctx, body, origlen)
 			if err != nil {
 				numFailures++

logtail/logtail_test.go

@@ -15,6 +15,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"testing/synctest"
 	"time"
@@ -666,6 +667,54 @@ func TestLoggerSetEnabled(t *testing.T) {
 	}
 }
 
+func TestLoggerSetEnabledStopsPendingRetry(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		var calls atomic.Int32
+
+		httpc := &http.Client{
+			Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
+				calls.Add(1)
+				return &http.Response{
+					StatusCode: http.StatusServiceUnavailable,
+					Header: http.Header{
+						"Retry-After": []string{"1"},
+					},
+					Body: io.NopCloser(strings.NewReader("try again")),
+				}, nil
+			}),
+		}
+
+		lg := NewLogger(Config{
+			BaseURL:      "http://logtail.test.invalid",
+			HTTPC:        httpc,
+			Bus:          eventbustest.NewBus(t),
+			FlushDelayFn: func() time.Duration { return 0 },
+		}, t.Logf)
+		defer func() {
+			ctx, cancel := context.WithCancel(context.Background())
+			cancel()
+			lg.Shutdown(ctx)
+		}()
+
+		lg.Logf("hello")
+
+		synctest.Wait()
+
+		if got := calls.Load(); got != 1 {
+			t.Fatalf("upload attempts before disabling = %d; want 1", got)
+		}
+
+		lg.SetEnabled(false)
+
+		time.Sleep(time.Second)
+		synctest.Wait()
+
+		if got := calls.Load(); got != 1 {
+			t.Fatalf("upload retried after SetEnabled(false): got %d attempts, want 1", got)
+		}
+	})
+}
+
 func TestAppendMetadata(t *testing.T) {
 	var lg Logger
 	lg.clock = tstest.NewClock(tstest.ClockOpts{Start: time.Date(2000, 01, 01, 0, 0, 0, 0, time.UTC)})