diff --git a/core/application/startup.go b/core/application/startup.go
index 1382835f6..a700615e2 100644
--- a/core/application/startup.go
+++ b/core/application/startup.go
@@ -259,7 +259,7 @@ func New(opts ...config.AppOption) (*Application, error) {
application.failoverManager = failover.New(application.ModelConfigLoader(),
failover.WithProber(failover.NewProber(func(ctx context.Context, cfg config.ModelConfig) (grpc.Backend, error) {
return application.ModelLoader().Load(backend.ModelOptions(cfg, options)...)
- }, application.ModelLoader().ModelPath)),
+ })),
failover.WithOnWarmChanged(application.applyFailoverWarmTargets),
)
diff --git a/core/services/failover/prober.go b/core/services/failover/prober.go
index b3055c17f..592d9ca41 100644
--- a/core/services/failover/prober.go
+++ b/core/services/failover/prober.go
@@ -8,12 +8,9 @@ import (
"errors"
"fmt"
"io"
- "io/fs"
"mime/multipart"
"net/http"
"net/url"
- "os"
- "path/filepath"
"strings"
"github.com/mudler/LocalAI/core/config"
@@ -27,13 +24,12 @@ type LoadFunc func(ctx context.Context, cfg config.ModelConfig) (grpc.Backend, e
// DefaultProber probes remote targets over the upstream's OpenAI-compatible
// API and local targets through their gRPC backend.
type DefaultProber struct {
- HTTP *http.Client
- Load LoadFunc
- ModelPath string
+ HTTP *http.Client
+ Load LoadFunc
}
-func NewProber(load LoadFunc, modelPath string) *DefaultProber {
- return &DefaultProber{HTTP: &http.Client{}, Load: load, ModelPath: modelPath}
+func NewProber(load LoadFunc) *DefaultProber {
+ return &DefaultProber{HTTP: &http.Client{}, Load: load}
}
func (p *DefaultProber) Liveness(ctx context.Context, cfg config.ModelConfig, kind Kind, warm bool) error {
@@ -43,7 +39,13 @@ func (p *DefaultProber) Liveness(ctx context.Context, cfg config.ModelConfig, ki
case warm:
return p.localHealth(ctx, cfg)
}
- return p.coldLiveness(cfg)
+ // A cold target is judged only by real requests: loading it only to probe
+ // it could evict other models, and no cheaper check is reliable. A missing
+ // model file is not one: models download on first use, some backends need
+ // no file, and a dotted name like "Phi-3.5-mini" looks like a file path. A
+ // false "down" would take away the very fallback the chain exists for,
+ // while a real load failure still trips the target and the request moves on.
+ return nil
}
func (p *DefaultProber) Inference(ctx context.Context, cfg config.ModelConfig, kind Kind, warm bool) error {
@@ -262,22 +264,3 @@ func (p *DefaultProber) localInference(ctx context.Context, cfg config.ModelConf
// transcription, so a real request adds little here.
return p.localHealth(ctx, cfg)
}
-
-// coldLiveness checks the model file without loading the model.
-func (p *DefaultProber) coldLiveness(cfg config.ModelConfig) error {
- f := cfg.Model
- if f == "" || p.ModelPath == "" || strings.Contains(f, "://") {
- return nil
- }
- path := f
- if !filepath.IsAbs(path) {
- path = filepath.Join(p.ModelPath, f)
- }
- if _, err := os.Stat(path); err != nil {
- if errors.Is(err, fs.ErrNotExist) && filepath.Ext(f) == "" {
- return nil // a repository id, downloaded on demand
- }
- return fmt.Errorf("model file %s: %w", f, err)
- }
- return nil
-}
diff --git a/core/services/failover/prober_test.go b/core/services/failover/prober_test.go
index 2b8b7f42d..0f65f8dc4 100644
--- a/core/services/failover/prober_test.go
+++ b/core/services/failover/prober_test.go
@@ -7,8 +7,6 @@ import (
"io"
"net/http"
"net/http/httptest"
- "os"
- "path/filepath"
"sync"
"github.com/mudler/LocalAI/core/config"
@@ -77,7 +75,7 @@ var _ = Describe("DefaultProber", func() {
BeforeEach(func() {
up = newFakeUpstream()
DeferCleanup(up.srv.Close)
- p = NewProber(nil, "")
+ p = NewProber(nil)
})
proxied := func(name, upstreamModel string, usecases ...string) config.ModelConfig {
@@ -140,7 +138,7 @@ var _ = Describe("DefaultProber", func() {
It("uses HealthCheck for warm local liveness and Predict for local chat inference", func() {
b := &fakeBackend{healthy: true}
- p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) { return b, nil }, "")
+ p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) { return b, nil })
c := config.ModelConfig{Name: "gemma", Backend: "llama-cpp", KnownUsecaseStrings: []string{"chat"}}
c.KnownUsecases = config.GetUsecasesFromYAML(c.KnownUsecaseStrings)
Expect(p.Liveness(ctx, c, KindLocal, true)).To(Succeed())
@@ -152,18 +150,18 @@ var _ = Describe("DefaultProber", func() {
Expect(p.Inference(ctx, c, KindLocal, true)).To(HaveOccurred())
})
- It("checks the model file for cold local liveness without loading", func() {
- dir := GinkgoT().TempDir()
+ It("passes cold local liveness without a model file and without loading", func() {
p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) {
Fail("cold liveness must not load the model")
return nil, nil
- }, dir)
- c := config.ModelConfig{Name: "cold", Backend: "llama-cpp"}
- c.Model = "weights.gguf"
- Expect(p.Liveness(ctx, c, KindLocal, false)).To(HaveOccurred())
- Expect(os.WriteFile(filepath.Join(dir, "weights.gguf"), []byte("x"), 0o600)).To(Succeed())
- Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed())
- c.Model = "org/some-hf-repo" // no extension: downloaded on demand
- Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed())
+ })
+ // None of these files exist: a missing file says nothing about whether
+ // the target can serve (download on first use, dotted names, backends
+ // that need no file). Only a real request may trip a cold target.
+ for _, model := range []string{"weights.gguf", "Phi-3.5-mini", "org/some-hf-repo", ""} {
+ c := config.ModelConfig{Name: "cold", Backend: "llama-cpp"}
+ c.Model = model
+ Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed(), model)
+ }
})
})
diff --git a/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md b/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md
index 11ef80e4f..551af9ab7 100644
--- a/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md
+++ b/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md
@@ -152,7 +152,7 @@ Chain states:
|---|---|---|
| remote | `GET /v1/models` returns 2xx and lists the upstream model. `` is the scheme and host of `proxy.upstream_url` plus any path prefix before `/v1`. The upstream model is `proxy.upstream_model`, or the target name when it is empty. `/v1/models` works on any OpenAI-compatible upstream, and `/readyz` exists only on LocalAI. | one minimal real request, chosen by usecase |
| local, `warm: true` | gRPC `HealthCheck` on the loaded backend. If the backend is not loaded (it crashed), a reload is the recovery attempt. | chat and completion: `Predict` with 1 token; embeddings: `Embedding` of `"ping"`; other usecases: `HealthCheck`. A local backend process that answers `HealthCheck` rarely fails only for TTS or transcription. |
-| local, cold | the model file exists (skipped for URLs and repository ids). The model is never loaded only to probe it. | none. After a trip, the target returns to `healthy` when `min_dwell` has passed. The next real request is the test. |
+| local, cold | none: a cold target is judged only by real requests; it is never loaded only to probe it. | none. After a trip, the target returns to `healthy` when `min_dwell` has passed. The next real request is the test. |
Minimal requests by usecase:
diff --git a/tests/e2e/e2e_failover_test.go b/tests/e2e/e2e_failover_test.go
index 540ea73c5..67e1227b4 100644
--- a/tests/e2e/e2e_failover_test.go
+++ b/tests/e2e/e2e_failover_test.go
@@ -23,18 +23,18 @@ var _ = Describe("Failover chains", Label("failover"), func() {
Expect(err).ToNot(HaveOccurred())
return resp
}
- expectServedByFallback := func(resp *http.Response) {
+ expectServedByMock := func(resp *http.Response) {
defer func() { _ = resp.Body.Close() }()
body, _ := io.ReadAll(resp.Body)
Expect(resp.StatusCode).To(BeNumerically("<", 300), "%s\nheaders: %v", body, resp.Header)
- Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("failover-fallback"))
+ Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("mock-model"))
Expect(resp.Header.Get("X-LocalAI-Failover")).To(Equal("fallback"))
}
// The entry name is the chain suffix: chain- is written by the suite.
DescribeTable("retries every endpoint family on the next target",
func(path string, body func(model string) map[string]any) {
- expectServedByFallback(postJSON(path, body("chain-"+CurrentSpecReport().LeafNodeText)))
+ expectServedByMock(postJSON(path, body("chain-"+CurrentSpecReport().LeafNodeText)))
},
Entry("chat", "/chat/completions", func(m string) map[string]any {
return map[string]any{"model": m, "messages": []map[string]string{{"role": "user", "content": "hi"}}}
@@ -71,7 +71,7 @@ var _ = Describe("Failover chains", Label("failover"), func() {
Expect(mw.Close()).To(Succeed())
resp, err := http.Post(apiURL+"/audio/transcriptions", mw.FormDataContentType(), &body)
Expect(err).ToNot(HaveOccurred())
- expectServedByFallback(resp)
+ expectServedByMock(resp)
})
Describe("remote targets", Ordered, func() {
diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go
index b27d25f64..a89318dfc 100644
--- a/tests/e2e/e2e_suite_test.go
+++ b/tests/e2e/e2e_suite_test.go
@@ -120,17 +120,7 @@ var _ = BeforeSuite(func() {
// Failover chains, one per endpoint family: target 0 is a mock model whose
// load always fails (the mock rejects models named fail-load*), so every
- // request exercises the retry onto failover-fallback. The fallback's model
- // file must exist: the failover manager's liveness probe marks a local
- // target with a missing file down, and mock-model.bin is never created.
- Expect(os.WriteFile(filepath.Join(modelsPath, "failover-fallback.bin"), nil, 0644)).To(Succeed())
- fallbackData, err := yaml.Marshal(map[string]any{
- "name": "failover-fallback",
- "backend": "mock-backend",
- "parameters": map[string]any{"model": "failover-fallback.bin"},
- })
- Expect(err).ToNot(HaveOccurred())
- Expect(os.WriteFile(filepath.Join(modelsPath, "failover-fallback.yaml"), fallbackData, 0644)).To(Succeed())
+ // request exercises the retry onto mock-model.
for _, family := range []string{"chat", "completion", "embeddings", "transcription", "tts", "image", "rerank", "vad"} {
for _, cfg := range []map[string]any{
{
@@ -141,7 +131,7 @@ var _ = BeforeSuite(func() {
{
"name": "chain-" + family,
"failover": map[string]any{
- "targets": []map[string]any{{"model": "fail-" + family}, {"model": "failover-fallback"}},
+ "targets": []map[string]any{{"model": "fail-" + family}, {"model": "mock-model"}},
},
},
} {