mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-29 17:44:30 -04:00
fix(failover): spill 429 admission rejections to the next target
#12113 changed admission control to reject with 429 instead of 503. failoverWriter only held back responses with status >= 500, so a 429 rejection reached the client and the chain never spilled to its next target. Hold 429 as well. An admission rejection is still flagged and spills without tripping the target. Any other 429 is not retryable, so it is released to the client unchanged. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude:claude-opus-5-5 [Claude Code]
This commit is contained in:
1 parent
9c156656bd
commit
d8571a8ee9
2 files changed
+17
-3
No files matched your search
@@ -220,8 +220,10 @@ func resetResponse(resp *echo.Response, base http.Header) {
|
||||
resp.Size = 0
|
||||
}
|
||||
|
||||
// failoverWriter holds back an error response (status >= 500) of a chain
|
||||
// request until the handler returns, so the retry can drop it.
|
||||
// failoverWriter holds back an error response of a chain request until the
|
||||
// handler returns, so the retry can drop it. It holds every 5xx, and 429 too
|
||||
// because admission control rejects with 429 and that must spill to the next
|
||||
// target; any other 429 is not retryable and is released unchanged.
|
||||
type failoverWriter struct {
|
||||
http.ResponseWriter
|
||||
active func() bool
|
||||
@@ -236,7 +238,7 @@ func (w *failoverWriter) WriteHeader(code int) {
|
||||
if w.held != 0 {
|
||||
return
|
||||
}
|
||||
if !w.committed && code >= 500 && w.active() {
|
||||
if !w.committed && (code >= 500 || code == http.StatusTooManyRequests) && w.active() {
|
||||
w.held = code
|
||||
return
|
||||
}
|
||||
|
||||
@@ -200,6 +200,18 @@ var _ = Describe("failover chains in the request pipeline", func() {
|
||||
Expect(st.Targets[0].State).To(Equal(failover.StateHealthy))
|
||||
})
|
||||
|
||||
It("sends a handler's own 429 as is, without retrying or tripping", func() {
|
||||
behavior["a"] = func(c echo.Context) error {
|
||||
return c.JSON(http.StatusTooManyRequests, map[string]string{"error": "slow down"})
|
||||
}
|
||||
rec := chat("chain")
|
||||
Expect(rec.Code).To(Equal(http.StatusTooManyRequests))
|
||||
Expect(rec.Body.String()).To(ContainSubstring("slow down"))
|
||||
Expect(calls).To(Equal([]string{"a"}))
|
||||
st, _ := fm.ChainStatus("chain")
|
||||
Expect(st.Targets[0].State).To(Equal(failover.StateHealthy))
|
||||
})
|
||||
|
||||
It("does not retry when the client cancelled", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
behavior["a"] = func(echo.Context) error { cancel(); return context.Canceled }
|
||||
|
||||
Reference in new issue
Block a user