mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-08 07:18:31 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(model): debounce model loads after a failure to stop retry-storms (#10728)

Browse Source 
A client that keeps polling a model whose load fails (e.g. a backend that
crashes deterministically on init) triggered a fresh backend start on
every request: request -> load -> crash in ~10s -> 500, repeat on the
next poll. Each attempt could leak GPU/CUDA state, and under
LOCALAI_SINGLE_ACTIVE_BACKEND it kept stealing the active slot from
healthy models. The existing loading-coalesce map only dedups
*concurrent* loads, so sequential polls were never covered.

Track load failures per modelID in ModelLoader. After a load fails,
refuse fresh load triggers for that model until a cooldown elapses,
returning a typed ModelLoadCooldownError that the HTTP layer maps to 503
with a Retry-After header. The cooldown grows exponentially per
consecutive failure (base, doubling, capped at 5m) and resets on a
successful load. The coalesced follower-retry of an in-flight burst
bypasses the gate, so a genuinely concurrent burst still gets its one
retry -- only new, independent triggers are refused, matching the
report's "refuse new load-triggers" wording.

Configurable via --model-load-failure-cooldown /
LOCALAI_MODEL_LOAD_FAILURE_COOLDOWN (default 10s, 0 disables), plumbed
through ApplicationConfig and applied unconditionally at startup.

Closes #10719


Assisted-by: Claude:claude-opus-4-8 [Claude Code]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
LocalAI [bot]
2026-07-07 22:55:24 +02:00
committed by GitHub
parent 1d5139f0a0
commit 97175f4b5a
7 changed files with 271 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
pkg/model/loader.go
View File

@@ -82,6 +82,33 @@ type ModelLoader struct {
// the exit code can't, since a child killed by our own SIGTERM/SIGKILL
// reports -1, indistinguishable from a signal-induced crash.
stoppingProcs sync.Map
// loadFailures records, per modelID, the cooldown window applied after a
// failed load so that a client repeatedly polling a broken model does not
// spawn (and leak) a fresh backend process on every request. Guarded by mu.
loadFailures map[string]*loadFailureState
loadFailureBaseCooldown time.Duration // first cooldown after a failure
loadFailureMaxCooldown time.Duration // cap for the exponential backoff
}
// loadFailureState tracks consecutive load failures for a single modelID and
// the instant at which its cooldown window expires.
type loadFailureState struct {
consecutive int
cooldownUntil time.Time
}
// ModelLoadCooldownError is returned when a model load is skipped because a
// recent attempt failed and the per-model cooldown window has not yet elapsed.
// The HTTP layer maps it to 503 with a Retry-After header so a polling client
// backs off instead of triggering a fresh backend start on every request.
type ModelLoadCooldownError struct {
ModelID string
RetryAfter time.Duration
}
func (e *ModelLoadCooldownError) Error() string {
return fmt.Sprintf("model %q load is in cooldown after a recent failure; retry after %s",
e.ModelID, e.RetryAfter.Round(time.Second))
}
// NewModelLoader creates a new ModelLoader instance.
@@ -95,11 +122,79 @@ func NewModelLoader(system *system.SystemState) *ModelLoader {
lruEvictionMaxRetries: 30, // Default: 30 retries
lruEvictionRetryInterval: 1 * time.Second, // Default: 1 second
backendLogs: NewBackendLogStore(1000),
loadFailures: make(map[string]*loadFailureState),
loadFailureBaseCooldown: 10 * time.Second, // Default: 10s after the first failure
loadFailureMaxCooldown: 5 * time.Minute, // Default cap for the backoff
}
return nml
}
// SetLoadFailureCooldown configures the per-model load-failure backoff. base is
// the cooldown applied after the first failure; it doubles on each consecutive
// failure up to max. base is authoritative (base <= 0 disables the cooldown
// entirely); max only overrides the current cap when > 0.
func (ml *ModelLoader) SetLoadFailureCooldown(base, max time.Duration) {
ml.mu.Lock()
defer ml.mu.Unlock()
if base < 0 {
base = 0
}
ml.loadFailureBaseCooldown = base
if max > 0 {
ml.loadFailureMaxCooldown = max
}
if ml.loadFailureMaxCooldown < ml.loadFailureBaseCooldown {
ml.loadFailureMaxCooldown = ml.loadFailureBaseCooldown
}
}
// cooldownRemaining returns how long the modelID's load cooldown still has to
// run, or 0 if there is none. Callers must hold ml.mu.
func (ml *ModelLoader) cooldownRemaining(modelID string) time.Duration {
st := ml.loadFailures[modelID]
if st == nil {
return 0
}
if remaining := time.Until(st.cooldownUntil); remaining > 0 {
return remaining
}
return 0
}
// recordLoadFailure grows the modelID's consecutive-failure count and arms the
// next cooldown window using exponential backoff capped at loadFailureMaxCooldown.
func (ml *ModelLoader) recordLoadFailure(modelID string) {
ml.mu.Lock()
defer ml.mu.Unlock()
if ml.loadFailureBaseCooldown <= 0 {
return // cooldown disabled
}
st := ml.loadFailures[modelID]
if st == nil {
st = &loadFailureState{}
ml.loadFailures[modelID] = st
}
st.consecutive++
// base * 2^(consecutive-1), clamped. Cap the shift to avoid overflowing
// the Duration; anything past the cap collapses to loadFailureMaxCooldown.
shift := st.consecutive - 1
if shift > 20 {
shift = 20
}
backoff := ml.loadFailureBaseCooldown * (1 << shift)
if backoff <= 0 || backoff > ml.loadFailureMaxCooldown {
backoff = ml.loadFailureMaxCooldown
}
st.cooldownUntil = time.Now().Add(backoff)
}
// clearLoadFailure resets the modelID's failure state after a successful load.
// Callers must hold ml.mu.
func (ml *ModelLoader) clearLoadFailure(modelID string) {
delete(ml.loadFailures, modelID)
}
// GetLoadingCount returns the number of models currently being loaded
func (ml *ModelLoader) GetLoadingCount() int {
ml.mu.Lock()
@@ -302,6 +397,14 @@ func (ml *ModelLoader) ListLoadedModels() []*Model {
}
func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string, string, string) (*Model, error)) (*Model, error) {
return ml.loadModel(modelID, modelName, loader, true)
}
// loadModel is the implementation behind LoadModel. checkCooldown gates fresh,
// independent load triggers behind the per-model failure cooldown; it is set to
// false for the coalesced retry of an in-flight burst (a follower whose leader
// just failed), which is not a new trigger and should still get its one retry.
func (ml *ModelLoader) loadModel(modelID, modelName string, loader func(string, string, string) (*Model, error), checkCooldown bool) (*Model, error) {
ml.mu.Lock()
distributed := ml.modelRouter != nil
ml.mu.Unlock()
@@ -353,6 +456,19 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
return model, nil
}
// If a recent load attempt for this model failed, short-circuit fresh load
// triggers until the cooldown elapses. This stops a client that keeps
// polling a broken model from spawning (and leaking) a new backend process
// on every request. The coalesced follower-retry below passes
// checkCooldown=false so an in-flight burst still gets its one retry.
if checkCooldown {
if retryAfter := ml.cooldownRemaining(modelID); retryAfter > 0 {
ml.mu.Unlock()
xlog.Debug("Model load in cooldown after a recent failure", "modelID", modelID, "retryAfter", retryAfter)
return nil, &ModelLoadCooldownError{ModelID: modelID, RetryAfter: retryAfter}
}
}
// Check if another goroutine is already loading this model
if loadingChan, isLoading := ml.loading[modelID]; isLoading {
ml.mu.Unlock()
@@ -366,8 +482,9 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
if model != nil {
return model, nil
}
// If still not loaded, the other goroutine failed - we'll try again
return ml.LoadModel(modelID, modelName, loader)
// If still not loaded, the other goroutine failed. Retry once as part of
// this burst, bypassing the cooldown gate (we are not a new trigger).
return ml.loadModel(modelID, modelName, loader, false)
}
// Mark this model as loading (create a channel that will be closed when done)
@@ -389,15 +506,18 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
model, err := loader(modelID, modelName, modelFile)
if err != nil {
ml.recordLoadFailure(modelID)
return nil, fmt.Errorf("failed to load model with internal loader: %s", err)
}
if model == nil {
ml.recordLoadFailure(modelID)
return nil, fmt.Errorf("loader didn't return a model")
}
// Add to models map
// Add to models map and reset any prior failure cooldown for this model.
ml.mu.Lock()
ml.clearLoadFailure(modelID)
ml.store.Set(modelID, model)
ml.mu.Unlock()

93
pkg/model/loader_test.go
View File

@@ -255,6 +255,11 @@ var _ = Describe("ModelLoader", func() {
mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
count := atomic.AddInt32(&attemptCount, 1)
if count == 1 {
// Hold the loading slot so the second request coalesces as a
// follower before this leader fails. That follower then gets
// the one in-burst retry, which bypasses the failure cooldown
// (the cooldown only gates fresh, independent load triggers).
time.Sleep(50 * time.Millisecond)
return nil, errors.New("first attempt fails")
}
return model.NewModel(modelID, modelName, nil), nil
@@ -270,7 +275,7 @@ var _ = Describe("ModelLoader", func() {
m1, err1 = modelLoader.LoadModel("retry-model", "test.model", mockLoader)
})
// Give first goroutine a head start
// Give first goroutine a head start so it owns the loading slot.
time.Sleep(10 * time.Millisecond)
wg.Go(func() {
@@ -305,4 +310,90 @@ var _ = Describe("ModelLoader", func() {
Expect(modelLoader).ToNot(BeNil())
})
})
Context("Load failure cooldown", func() {
It("refuses a fresh load within the cooldown window without re-invoking the loader", func() {
modelLoader.SetLoadFailureCooldown(60*time.Millisecond, 240*time.Millisecond)
var loadCount int32
failing := func(modelID, modelName, modelFile string) (*model.Model, error) {
atomic.AddInt32(&loadCount, 1)
return nil, errors.New("boom")
}
// First attempt runs the loader and fails (not a cooldown error).
_, err := modelLoader.LoadModel("broken", "test.model", failing)
Expect(err).To(HaveOccurred())
var coolErr *model.ModelLoadCooldownError
Expect(errors.As(err, &coolErr)).To(BeFalse())
Expect(atomic.LoadInt32(&loadCount)).To(Equal(int32(1)))
// An immediate retry is short-circuited: cooldown error, loader untouched.
_, err = modelLoader.LoadModel("broken", "test.model", failing)
Expect(errors.As(err, &coolErr)).To(BeTrue())
Expect(coolErr.ModelID).To(Equal("broken"))
Expect(coolErr.RetryAfter).To(BeNumerically(">", time.Duration(0)))
Expect(atomic.LoadInt32(&loadCount)).To(Equal(int32(1)))
// Once the window elapses the loader is attempted again.
Eventually(func() int32 {
_, _ = modelLoader.LoadModel("broken", "test.model", failing)
return atomic.LoadInt32(&loadCount)
}, "1s", "20ms").Should(BeNumerically(">=", 2))
})
It("clears the cooldown after a successful load", func() {
modelLoader.SetLoadFailureCooldown(60*time.Millisecond, 240*time.Millisecond)
var attempts int32
loader := func(modelID, modelName, modelFile string) (*model.Model, error) {
if atomic.AddInt32(&attempts, 1) == 1 {
return nil, errors.New("boom")
}
m := model.NewModel(modelID, modelName, nil)
m.MarkHealthy()
return m, nil
}
_, err := modelLoader.LoadModel("flaky", "test.model", loader)
Expect(err).To(HaveOccurred())
// After the window, the retry succeeds and resets the failure state.
var m *model.Model
Eventually(func() error {
m, err = modelLoader.LoadModel("flaky", "test.model", loader)
return err
}, "1s", "20ms").Should(Succeed())
Expect(m).ToNot(BeNil())
// A subsequent load returns the cached model, never a cooldown error.
m2, err := modelLoader.LoadModel("flaky", "test.model", loader)
Expect(err).ToNot(HaveOccurred())
Expect(m2).To(Equal(m))
})
It("grows the cooldown on consecutive failures", func() {
modelLoader.SetLoadFailureCooldown(50*time.Millisecond, 10*time.Second)
failing := func(modelID, modelName, modelFile string) (*model.Model, error) {
return nil, errors.New("boom")
}
// Failure 1, then read its cooldown.
_, err := modelLoader.LoadModel("bad", "test.model", failing)
Expect(err).To(HaveOccurred())
_, err = modelLoader.LoadModel("bad", "test.model", failing)
var c1 *model.ModelLoadCooldownError
Expect(errors.As(err, &c1)).To(BeTrue())
// Wait out the first window, trigger failure 2, read its (larger) cooldown.
time.Sleep(70 * time.Millisecond)
_, err = modelLoader.LoadModel("bad", "test.model", failing)
Expect(err).To(HaveOccurred())
_, err = modelLoader.LoadModel("bad", "test.model", failing)
var c2 *model.ModelLoadCooldownError
Expect(errors.As(err, &c2)).To(BeTrue())
Expect(c2.RetryAfter).To(BeNumerically(">", c1.RetryAfter))
})
})
})