diff --git a/core/application/startup.go b/core/application/startup.go index 1382835f6..a700615e2 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -259,7 +259,7 @@ func New(opts ...config.AppOption) (*Application, error) { application.failoverManager = failover.New(application.ModelConfigLoader(), failover.WithProber(failover.NewProber(func(ctx context.Context, cfg config.ModelConfig) (grpc.Backend, error) { return application.ModelLoader().Load(backend.ModelOptions(cfg, options)...) - }, application.ModelLoader().ModelPath)), + })), failover.WithOnWarmChanged(application.applyFailoverWarmTargets), ) diff --git a/core/services/failover/prober.go b/core/services/failover/prober.go index b3055c17f..592d9ca41 100644 --- a/core/services/failover/prober.go +++ b/core/services/failover/prober.go @@ -8,12 +8,9 @@ import ( "errors" "fmt" "io" - "io/fs" "mime/multipart" "net/http" "net/url" - "os" - "path/filepath" "strings" "github.com/mudler/LocalAI/core/config" @@ -27,13 +24,12 @@ type LoadFunc func(ctx context.Context, cfg config.ModelConfig) (grpc.Backend, e // DefaultProber probes remote targets over the upstream's OpenAI-compatible // API and local targets through their gRPC backend. type DefaultProber struct { - HTTP *http.Client - Load LoadFunc - ModelPath string + HTTP *http.Client + Load LoadFunc } -func NewProber(load LoadFunc, modelPath string) *DefaultProber { - return &DefaultProber{HTTP: &http.Client{}, Load: load, ModelPath: modelPath} +func NewProber(load LoadFunc) *DefaultProber { + return &DefaultProber{HTTP: &http.Client{}, Load: load} } func (p *DefaultProber) Liveness(ctx context.Context, cfg config.ModelConfig, kind Kind, warm bool) error { @@ -43,7 +39,13 @@ func (p *DefaultProber) Liveness(ctx context.Context, cfg config.ModelConfig, ki case warm: return p.localHealth(ctx, cfg) } - return p.coldLiveness(cfg) + // A cold target is judged only by real requests: loading it only to probe + // it could evict other models, and no cheaper check is reliable. A missing + // model file is not one: models download on first use, some backends need + // no file, and a dotted name like "Phi-3.5-mini" looks like a file path. A + // false "down" would take away the very fallback the chain exists for, + // while a real load failure still trips the target and the request moves on. + return nil } func (p *DefaultProber) Inference(ctx context.Context, cfg config.ModelConfig, kind Kind, warm bool) error { @@ -262,22 +264,3 @@ func (p *DefaultProber) localInference(ctx context.Context, cfg config.ModelConf // transcription, so a real request adds little here. return p.localHealth(ctx, cfg) } - -// coldLiveness checks the model file without loading the model. -func (p *DefaultProber) coldLiveness(cfg config.ModelConfig) error { - f := cfg.Model - if f == "" || p.ModelPath == "" || strings.Contains(f, "://") { - return nil - } - path := f - if !filepath.IsAbs(path) { - path = filepath.Join(p.ModelPath, f) - } - if _, err := os.Stat(path); err != nil { - if errors.Is(err, fs.ErrNotExist) && filepath.Ext(f) == "" { - return nil // a repository id, downloaded on demand - } - return fmt.Errorf("model file %s: %w", f, err) - } - return nil -} diff --git a/core/services/failover/prober_test.go b/core/services/failover/prober_test.go index 2b8b7f42d..0f65f8dc4 100644 --- a/core/services/failover/prober_test.go +++ b/core/services/failover/prober_test.go @@ -7,8 +7,6 @@ import ( "io" "net/http" "net/http/httptest" - "os" - "path/filepath" "sync" "github.com/mudler/LocalAI/core/config" @@ -77,7 +75,7 @@ var _ = Describe("DefaultProber", func() { BeforeEach(func() { up = newFakeUpstream() DeferCleanup(up.srv.Close) - p = NewProber(nil, "") + p = NewProber(nil) }) proxied := func(name, upstreamModel string, usecases ...string) config.ModelConfig { @@ -140,7 +138,7 @@ var _ = Describe("DefaultProber", func() { It("uses HealthCheck for warm local liveness and Predict for local chat inference", func() { b := &fakeBackend{healthy: true} - p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) { return b, nil }, "") + p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) { return b, nil }) c := config.ModelConfig{Name: "gemma", Backend: "llama-cpp", KnownUsecaseStrings: []string{"chat"}} c.KnownUsecases = config.GetUsecasesFromYAML(c.KnownUsecaseStrings) Expect(p.Liveness(ctx, c, KindLocal, true)).To(Succeed()) @@ -152,18 +150,18 @@ var _ = Describe("DefaultProber", func() { Expect(p.Inference(ctx, c, KindLocal, true)).To(HaveOccurred()) }) - It("checks the model file for cold local liveness without loading", func() { - dir := GinkgoT().TempDir() + It("passes cold local liveness without a model file and without loading", func() { p = NewProber(func(context.Context, config.ModelConfig) (grpc.Backend, error) { Fail("cold liveness must not load the model") return nil, nil - }, dir) - c := config.ModelConfig{Name: "cold", Backend: "llama-cpp"} - c.Model = "weights.gguf" - Expect(p.Liveness(ctx, c, KindLocal, false)).To(HaveOccurred()) - Expect(os.WriteFile(filepath.Join(dir, "weights.gguf"), []byte("x"), 0o600)).To(Succeed()) - Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed()) - c.Model = "org/some-hf-repo" // no extension: downloaded on demand - Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed()) + }) + // None of these files exist: a missing file says nothing about whether + // the target can serve (download on first use, dotted names, backends + // that need no file). Only a real request may trip a cold target. + for _, model := range []string{"weights.gguf", "Phi-3.5-mini", "org/some-hf-repo", ""} { + c := config.ModelConfig{Name: "cold", Backend: "llama-cpp"} + c.Model = model + Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed(), model) + } }) }) diff --git a/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md b/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md index 11ef80e4f..551af9ab7 100644 --- a/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md +++ b/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md @@ -152,7 +152,7 @@ Chain states: |---|---|---| | remote | `GET /v1/models` returns 2xx and lists the upstream model. `` is the scheme and host of `proxy.upstream_url` plus any path prefix before `/v1`. The upstream model is `proxy.upstream_model`, or the target name when it is empty. `/v1/models` works on any OpenAI-compatible upstream, and `/readyz` exists only on LocalAI. | one minimal real request, chosen by usecase | | local, `warm: true` | gRPC `HealthCheck` on the loaded backend. If the backend is not loaded (it crashed), a reload is the recovery attempt. | chat and completion: `Predict` with 1 token; embeddings: `Embedding` of `"ping"`; other usecases: `HealthCheck`. A local backend process that answers `HealthCheck` rarely fails only for TTS or transcription. | -| local, cold | the model file exists (skipped for URLs and repository ids). The model is never loaded only to probe it. | none. After a trip, the target returns to `healthy` when `min_dwell` has passed. The next real request is the test. | +| local, cold | none: a cold target is judged only by real requests; it is never loaded only to probe it. | none. After a trip, the target returns to `healthy` when `min_dwell` has passed. The next real request is the test. | Minimal requests by usecase: diff --git a/tests/e2e/e2e_failover_test.go b/tests/e2e/e2e_failover_test.go index 540ea73c5..67e1227b4 100644 --- a/tests/e2e/e2e_failover_test.go +++ b/tests/e2e/e2e_failover_test.go @@ -23,18 +23,18 @@ var _ = Describe("Failover chains", Label("failover"), func() { Expect(err).ToNot(HaveOccurred()) return resp } - expectServedByFallback := func(resp *http.Response) { + expectServedByMock := func(resp *http.Response) { defer func() { _ = resp.Body.Close() }() body, _ := io.ReadAll(resp.Body) Expect(resp.StatusCode).To(BeNumerically("<", 300), "%s\nheaders: %v", body, resp.Header) - Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("failover-fallback")) + Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("mock-model")) Expect(resp.Header.Get("X-LocalAI-Failover")).To(Equal("fallback")) } // The entry name is the chain suffix: chain- is written by the suite. DescribeTable("retries every endpoint family on the next target", func(path string, body func(model string) map[string]any) { - expectServedByFallback(postJSON(path, body("chain-"+CurrentSpecReport().LeafNodeText))) + expectServedByMock(postJSON(path, body("chain-"+CurrentSpecReport().LeafNodeText))) }, Entry("chat", "/chat/completions", func(m string) map[string]any { return map[string]any{"model": m, "messages": []map[string]string{{"role": "user", "content": "hi"}}} @@ -71,7 +71,7 @@ var _ = Describe("Failover chains", Label("failover"), func() { Expect(mw.Close()).To(Succeed()) resp, err := http.Post(apiURL+"/audio/transcriptions", mw.FormDataContentType(), &body) Expect(err).ToNot(HaveOccurred()) - expectServedByFallback(resp) + expectServedByMock(resp) }) Describe("remote targets", Ordered, func() { diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index b27d25f64..a89318dfc 100644 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -120,17 +120,7 @@ var _ = BeforeSuite(func() { // Failover chains, one per endpoint family: target 0 is a mock model whose // load always fails (the mock rejects models named fail-load*), so every - // request exercises the retry onto failover-fallback. The fallback's model - // file must exist: the failover manager's liveness probe marks a local - // target with a missing file down, and mock-model.bin is never created. - Expect(os.WriteFile(filepath.Join(modelsPath, "failover-fallback.bin"), nil, 0644)).To(Succeed()) - fallbackData, err := yaml.Marshal(map[string]any{ - "name": "failover-fallback", - "backend": "mock-backend", - "parameters": map[string]any{"model": "failover-fallback.bin"}, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(os.WriteFile(filepath.Join(modelsPath, "failover-fallback.yaml"), fallbackData, 0644)).To(Succeed()) + // request exercises the retry onto mock-model. for _, family := range []string{"chat", "completion", "embeddings", "transcription", "tts", "image", "rerank", "vad"} { for _, cfg := range []map[string]any{ { @@ -141,7 +131,7 @@ var _ = BeforeSuite(func() { { "name": "chain-" + family, "failover": map[string]any{ - "targets": []map[string]any{{"model": "fail-" + family}, {"model": "failover-fallback"}}, + "targets": []map[string]any{{"model": "fail-" + family}, {"model": "mock-model"}}, }, }, } {