fix(failover): stop judging cold local targets by their model file

A missing file is not a reliable signal: models download on first use,
some backends need no file, and dotted names like Phi-3.5-mini look like
paths. Marking such a fallback down removed the retry a chain exists
for. Cold targets are now judged only by real requests.

Assisted-by: Claude:claude-opus-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/application/startup.go

@@ -259,7 +259,7 @@ func New(opts ...config.AppOption) (*Application, error) {
 	application.failoverManager = failover.New(application.ModelConfigLoader(),
 		failover.WithProber(failover.NewProber(func(ctx context.Context, cfg config.ModelConfig) (grpc.Backend, error) {
 			return application.ModelLoader().Load(backend.ModelOptions(cfg, options)...)
-		}, application.ModelLoader().ModelPath)),
+		})),
 		failover.WithOnWarmChanged(application.applyFailoverWarmTargets),
 	)
 

core/services/failover/prober.go

@@ -8,12 +8,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/fs"
 	"mime/multipart"
 	"net/http"
 	"net/url"
-	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/mudler/LocalAI/core/config"
@@ -27,13 +24,12 @@ type LoadFunc func(ctx context.Context, cfg config.ModelConfig) (grpc.Backend, e
 // DefaultProber probes remote targets over the upstream's OpenAI-compatible
 // API and local targets through their gRPC backend.
 type DefaultProber struct {
-	HTTP      *http.Client
-	Load      LoadFunc
-	ModelPath string
+	HTTP *http.Client
+	Load LoadFunc
 }
 
-func NewProber(load LoadFunc, modelPath string) *DefaultProber {
-	return &DefaultProber{HTTP: &http.Client{}, Load: load, ModelPath: modelPath}
+func NewProber(load LoadFunc) *DefaultProber {
+	return &DefaultProber{HTTP: &http.Client{}, Load: load}
 }
 
 func (p *DefaultProber) Liveness(ctx context.Context, cfg config.ModelConfig, kind Kind, warm bool) error {
@@ -43,7 +39,13 @@ func (p *DefaultProber) Liveness(ctx context.Context, cfg config.ModelConfig, ki
 	case warm:
 		return p.localHealth(ctx, cfg)
 	}
-	return p.coldLiveness(cfg)
+	// A cold target is judged only by real requests: loading it only to probe
+	// it could evict other models, and no cheaper check is reliable. A missing
+	// model file is not one: models download on first use, some backends need
+	// no file, and a dotted name like "Phi-3.5-mini" looks like a file path. A
+	// false "down" would take away the very fallback the chain exists for,
+	// while a real load failure still trips the target and the request moves on.
+	return nil
 }
 
 func (p *DefaultProber) Inference(ctx context.Context, cfg config.ModelConfig, kind Kind, warm bool) error {
@@ -262,22 +264,3 @@ func (p *DefaultProber) localInference(ctx context.Context, cfg config.ModelConf
 	// transcription, so a real request adds little here.
 	return p.localHealth(ctx, cfg)
 }
-
-// coldLiveness checks the model file without loading the model.
-func (p *DefaultProber) coldLiveness(cfg config.ModelConfig) error {
-	f := cfg.Model
-	if f == "" || p.ModelPath == "" || strings.Contains(f, "://") {
-		return nil
-	}
-	path := f
-	if !filepath.IsAbs(path) {
-		path = filepath.Join(p.ModelPath, f)
-	}
-	if _, err := os.Stat(path); err != nil {
-		if errors.Is(err, fs.ErrNotExist) && filepath.Ext(f) == "" {
-			return nil // a repository id, downloaded on demand
-		}
-		return fmt.Errorf("model file %s: %w", f, err)
-	}
-	return nil
-}

core/services/failover/prober_test.go

@@ -7,8 +7,6 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
-	"os"
-	"path/filepath"
 	"sync"
 
 	"github.com/mudler/LocalAI/core/config"
@@ -77,7 +75,7 @@ var _ = Describe("DefaultProber", func() {
 	BeforeEach(func() {
 		up = newFakeUpstream()
 		DeferCleanup(up.srv.Close)
-		p = NewProber(nil, "")
+		p = NewProber(nil)
 	})
 
 	proxied := func(name, upstreamModel string, usecases ...string) config.ModelConfig {
@@ -140,7 +138,7 @@ var _ = Describe("DefaultProber", func() {
 
 	It("uses HealthCheck for warm local liveness and Predict for local chat inference", func() {
 		b := &fakeBackend{healthy: true}
-		p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) { return b, nil }, "")
+		p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) { return b, nil })
 		c := config.ModelConfig{Name: "gemma", Backend: "llama-cpp", KnownUsecaseStrings: []string{"chat"}}
 		c.KnownUsecases = config.GetUsecasesFromYAML(c.KnownUsecaseStrings)
 		Expect(p.Liveness(ctx, c, KindLocal, true)).To(Succeed())
@@ -152,18 +150,18 @@ var _ = Describe("DefaultProber", func() {
 		Expect(p.Inference(ctx, c, KindLocal, true)).To(HaveOccurred())
 	})
 
-	It("checks the model file for cold local liveness without loading", func() {
-		dir := GinkgoT().TempDir()
+	It("passes cold local liveness without a model file and without loading", func() {
 		p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) {
 			Fail("cold liveness must not load the model")
 			return nil, nil
-		}, dir)
-		c := config.ModelConfig{Name: "cold", Backend: "llama-cpp"}
-		c.Model = "weights.gguf"
-		Expect(p.Liveness(ctx, c, KindLocal, false)).To(HaveOccurred())
-		Expect(os.WriteFile(filepath.Join(dir, "weights.gguf"), []byte("x"), 0o600)).To(Succeed())
-		Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed())
-		c.Model = "org/some-hf-repo" // no extension: downloaded on demand
-		Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed())
+		})
+		// None of these files exist: a missing file says nothing about whether
+		// the target can serve (download on first use, dotted names, backends
+		// that need no file). Only a real request may trip a cold target.
+		for _, model := range []string{"weights.gguf", "Phi-3.5-mini", "org/some-hf-repo", ""} {
+			c := config.ModelConfig{Name: "cold", Backend: "llama-cpp"}
+			c.Model = model
+			Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed(), model)
+		}
 	})
 })

docs/superpowers/specs/2026-09-26-model-failover-chains-design.md

@@ -152,7 +152,7 @@ Chain states:
 |---|---|---|
 | remote | `GET <base>/v1/models` returns 2xx and lists the upstream model. `<base>` is the scheme and host of `proxy.upstream_url` plus any path prefix before `/v1`. The upstream model is `proxy.upstream_model`, or the target name when it is empty. `/v1/models` works on any OpenAI-compatible upstream, and `/readyz` exists only on LocalAI. | one minimal real request, chosen by usecase |
 | local, `warm: true` | gRPC `HealthCheck` on the loaded backend. If the backend is not loaded (it crashed), a reload is the recovery attempt. | chat and completion: `Predict` with 1 token; embeddings: `Embedding` of `"ping"`; other usecases: `HealthCheck`. A local backend process that answers `HealthCheck` rarely fails only for TTS or transcription. |
-| local, cold | the model file exists (skipped for URLs and repository ids). The model is never loaded only to probe it. | none. After a trip, the target returns to `healthy` when `min_dwell` has passed. The next real request is the test. |
+| local, cold | none: a cold target is judged only by real requests; it is never loaded only to probe it. | none. After a trip, the target returns to `healthy` when `min_dwell` has passed. The next real request is the test. |
 
 Minimal requests by usecase:
 

tests/e2e/e2e_failover_test.go

@@ -23,18 +23,18 @@ var _ = Describe("Failover chains", Label("failover"), func() {
 		Expect(err).ToNot(HaveOccurred())
 		return resp
 	}
-	expectServedByFallback := func(resp *http.Response) {
+	expectServedByMock := func(resp *http.Response) {
 		defer func() { _ = resp.Body.Close() }()
 		body, _ := io.ReadAll(resp.Body)
 		Expect(resp.StatusCode).To(BeNumerically("<", 300), "%s\nheaders: %v", body, resp.Header)
-		Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("failover-fallback"))
+		Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("mock-model"))
 		Expect(resp.Header.Get("X-LocalAI-Failover")).To(Equal("fallback"))
 	}
 
 	// The entry name is the chain suffix: chain-<name> is written by the suite.
 	DescribeTable("retries every endpoint family on the next target",
 		func(path string, body func(model string) map[string]any) {
-			expectServedByFallback(postJSON(path, body("chain-"+CurrentSpecReport().LeafNodeText)))
+			expectServedByMock(postJSON(path, body("chain-"+CurrentSpecReport().LeafNodeText)))
 		},
 		Entry("chat", "/chat/completions", func(m string) map[string]any {
 			return map[string]any{"model": m, "messages": []map[string]string{{"role": "user", "content": "hi"}}}
@@ -71,7 +71,7 @@ var _ = Describe("Failover chains", Label("failover"), func() {
 		Expect(mw.Close()).To(Succeed())
 		resp, err := http.Post(apiURL+"/audio/transcriptions", mw.FormDataContentType(), &body)
 		Expect(err).ToNot(HaveOccurred())
-		expectServedByFallback(resp)
+		expectServedByMock(resp)
 	})
 
 	Describe("remote targets", Ordered, func() {

tests/e2e/e2e_suite_test.go

@@ -120,17 +120,7 @@ var _ = BeforeSuite(func() {
 
 	// Failover chains, one per endpoint family: target 0 is a mock model whose
 	// load always fails (the mock rejects models named fail-load*), so every
-	// request exercises the retry onto failover-fallback. The fallback's model
-	// file must exist: the failover manager's liveness probe marks a local
-	// target with a missing file down, and mock-model.bin is never created.
-	Expect(os.WriteFile(filepath.Join(modelsPath, "failover-fallback.bin"), nil, 0644)).To(Succeed())
-	fallbackData, err := yaml.Marshal(map[string]any{
-		"name":       "failover-fallback",
-		"backend":    "mock-backend",
-		"parameters": map[string]any{"model": "failover-fallback.bin"},
-	})
-	Expect(err).ToNot(HaveOccurred())
-	Expect(os.WriteFile(filepath.Join(modelsPath, "failover-fallback.yaml"), fallbackData, 0644)).To(Succeed())
+	// request exercises the retry onto mock-model.
 	for _, family := range []string{"chat", "completion", "embeddings", "transcription", "tts", "image", "rerank", "vad"} {
 		for _, cfg := range []map[string]any{
 			{
@@ -141,7 +131,7 @@ var _ = BeforeSuite(func() {
 			{
 				"name": "chain-" + family,
 				"failover": map[string]any{
-					"targets": []map[string]any{{"model": "fail-" + family}, {"model": "failover-fallback"}},
+					"targets": []map[string]any{{"model": "fail-" + family}, {"model": "mock-model"}},
 				},
 			},
 		} {