diff --git a/core/http/middleware/failover.go b/core/http/middleware/failover.go index e2df6285f..8c68e7de4 100644 --- a/core/http/middleware/failover.go +++ b/core/http/middleware/failover.go @@ -220,8 +220,10 @@ func resetResponse(resp *echo.Response, base http.Header) { resp.Size = 0 } -// failoverWriter holds back an error response (status >= 500) of a chain -// request until the handler returns, so the retry can drop it. +// failoverWriter holds back an error response of a chain request until the +// handler returns, so the retry can drop it. It holds every 5xx, and 429 too +// because admission control rejects with 429 and that must spill to the next +// target; any other 429 is not retryable and is released unchanged. type failoverWriter struct { http.ResponseWriter active func() bool @@ -236,7 +238,7 @@ func (w *failoverWriter) WriteHeader(code int) { if w.held != 0 { return } - if !w.committed && code >= 500 && w.active() { + if !w.committed && (code >= 500 || code == http.StatusTooManyRequests) && w.active() { w.held = code return } diff --git a/core/http/middleware/failover_test.go b/core/http/middleware/failover_test.go index 17625621f..5c42b93dd 100644 --- a/core/http/middleware/failover_test.go +++ b/core/http/middleware/failover_test.go @@ -200,6 +200,18 @@ var _ = Describe("failover chains in the request pipeline", func() { Expect(st.Targets[0].State).To(Equal(failover.StateHealthy)) }) + It("sends a handler's own 429 as is, without retrying or tripping", func() { + behavior["a"] = func(c echo.Context) error { + return c.JSON(http.StatusTooManyRequests, map[string]string{"error": "slow down"}) + } + rec := chat("chain") + Expect(rec.Code).To(Equal(http.StatusTooManyRequests)) + Expect(rec.Body.String()).To(ContainSubstring("slow down")) + Expect(calls).To(Equal([]string{"a"})) + st, _ := fm.ChainStatus("chain") + Expect(st.Targets[0].State).To(Equal(failover.StateHealthy)) + }) + It("does not retry when the client cancelled", func() { ctx, cancel := context.WithCancel(context.Background()) behavior["a"] = func(echo.Context) error { cancel(); return context.Canceled }