From e61bd2d78ede9d1b44206b9dd9a6dcfb928b7df1 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 27 Sep 2026 02:29:04 +0000 Subject: [PATCH] fix(failover): do not trip targets on a gRPC message size limit ResourceExhausted is retried and trips the target, which is right for a rate limit or an out-of-memory backend. A payload over the gRPC message cap is ResourceExhausted too, but every target rejects it the same way, so it tripped the whole chain. Classify it as a request error. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Ettore Di Giacinto --- core/services/failover/classify.go | 5 +++++ core/services/failover/classify_test.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/core/services/failover/classify.go b/core/services/failover/classify.go index ec669a402..ac499fa69 100644 --- a/core/services/failover/classify.go +++ b/core/services/failover/classify.go @@ -22,6 +22,11 @@ var requestErrorMarkers = []string{ "exceeds the available context size", "is larger than the max context size", "maximum context length", + // A payload over the gRPC message cap is ResourceExhausted too, but it + // is the request's size, not the target's load: every target has the + // same cap. + "larger than max", + "gRPC message limit exceeded", } // IsRetryable reports whether a failed attempt should move to the next diff --git a/core/services/failover/classify_test.go b/core/services/failover/classify_test.go index d2fd90381..08bc05f07 100644 --- a/core/services/failover/classify_test.go +++ b/core/services/failover/classify_test.go @@ -36,6 +36,10 @@ var _ = DescribeTable("IsRetryable", Entry("cloud-proxy upstream 503", errors.New("cloud-proxy: upstream 503: no healthy nodes"), 0, true), Entry("cloud-proxy upstream 429 stays 4xx", errors.New("cloud-proxy: upstream 429: slow down"), 0, false), Entry("context overflow", errors.New("the request exceeds the available context size"), 0, false), + Entry("grpc received message over the size limit", grpcstatus.Error(codes.ResourceExhausted, "grpc: received message larger than max (60000000 vs. 52428800)"), 0, false), + Entry("grpc sent message over the size limit", grpcstatus.Error(codes.ResourceExhausted, "grpc: trying to send message larger than max (60000000 vs. 52428800)"), 0, false), + Entry("embeddings over the gRPC message limit", fmt.Errorf("conversation too long for per-token embeddings (gRPC message limit exceeded): %w", + grpcstatus.Error(codes.ResourceExhausted, "grpc: received message larger than max (1 vs. 0)")), 0, false), Entry("dial error", errors.New("dial tcp 10.0.0.1:8080: connect: connection refused"), 0, true), )