From e183ea11d1ba700dda42ca7a762508fa231d00a8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 26 Sep 2026 19:26:07 +0000 Subject: [PATCH] fix(failover): send remote targets their own upstream model A chain request reached a cloud-proxy target with the client's model, the chain name, whenever the target set no upstream_model: passthrough forwards the body's model and translate falls back to it. The upstream answered 404, which neither retries nor trips, while the liveness probe, which checks the target's own name, kept passing. PrepareTarget now sets the upstream model of a remote target to proxy.upstream_model or the target name, the same name the probe uses. The request pipeline and realtime chain stages both call it. Assisted-by: Claude:claude-opus-5-5 --- core/http/endpoints/openai/realtime_model.go | 20 ++++++++----- core/http/middleware/failover.go | 1 + core/http/middleware/failover_test.go | 30 +++++++++++++++++++ core/services/failover/prober.go | 11 +++++++ core/services/failover/prober_test.go | 17 +++++++++++ docs/content/features/model-failover.md | 3 ++ ...2026-09-26-model-failover-chains-design.md | 6 ++++ tests/e2e/e2e_failover_test.go | 17 +++++++++++ 8 files changed, 98 insertions(+), 7 deletions(-) diff --git a/core/http/endpoints/openai/realtime_model.go b/core/http/endpoints/openai/realtime_model.go index a90a0e4e5..ee42bde2d 100644 --- a/core/http/endpoints/openai/realtime_model.go +++ b/core/http/endpoints/openai/realtime_model.go @@ -1032,6 +1032,14 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model // takes the chain's active target, so everything that inspects stage // configs at session start (voice, reasoning, templates) sees a real model. stageChains := map[string]string{} + loadTarget := func(name string) (*config.ModelConfig, error) { + cfg, err := cl.LoadResolvedModelConfig(name, ml.ModelPath, appConfig.ToConfigLoaderOptions()...) + if err != nil { + return nil, err + } + failover.PrepareTarget(cfg) + return cfg, nil + } resolveStage := func(stage string, cfg *config.ModelConfig) (*config.ModelConfig, error) { if cfg == nil || !cfg.IsFailover() { return cfg, nil @@ -1044,7 +1052,7 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model return nil, fmt.Errorf("failover chain %q not found", cfg.Name) } stageChains[stage] = cfg.Name - return cl.LoadResolvedModelConfig(st.Active, ml.ModelPath, appConfig.ToConfigLoaderOptions()...) + return loadTarget(st.Active) } cfgVAD, err := cl.LoadResolvedModelConfig(pipeline.VAD, ml.ModelPath, appConfig.ToConfigLoaderOptions()...) @@ -1185,12 +1193,10 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model appConfig: appConfig, evaluator: evaluator, - stageChains: stageChains, - stageTargetConfig: func(name string) (*config.ModelConfig, error) { - return cl.LoadResolvedModelConfig(name, ml.ModelPath, appConfig.ToConfigLoaderOptions()...) - }, - tuneLLM: tuneLLM, - appTracing: appConfig.EnableTracing, + stageChains: stageChains, + stageTargetConfig: loadTarget, + tuneLLM: tuneLLM, + appTracing: appConfig.EnableTracing, } if routing != nil { wm.routerDeps = routing.Deps diff --git a/core/http/middleware/failover.go b/core/http/middleware/failover.go index f3744fdd9..853a3f22a 100644 --- a/core/http/middleware/failover.go +++ b/core/http/middleware/failover.go @@ -58,6 +58,7 @@ func (re *RequestExtractor) resolveFailover(c echo.Context, requested string, ch return nil, fmt.Errorf("failover chain %q: target %q is disabled", chain.Name, cfg.Name) } if err == nil { + failover.PrepareTarget(cfg) // cfg is a copy c.Set(ContextKeyRequestedModel, requested) c.Set(ContextKeyServedModel, cfg.Name) setFailoverHeaders(c.Response().Header(), st.attempt) diff --git a/core/http/middleware/failover_test.go b/core/http/middleware/failover_test.go index 81fff5e02..b5e8ba5b7 100644 --- a/core/http/middleware/failover_test.go +++ b/core/http/middleware/failover_test.go @@ -79,6 +79,9 @@ var _ = Describe("failover chains in the request pipeline", func() { write("off", "name: off\nbackend: fake-o\ndisabled: true\n") write("chain-capped", "name: chain-capped\nfailover:\n targets:\n - model: capped\n - model: b\n") write("chain-off", "name: chain-off\nfailover:\n targets:\n - model: off\n - model: b\n") + write("remote", "name: remote\nbackend: cloud-proxy\nproxy:\n mode: passthrough\n upstream_url: http://127.0.0.1:1/v1/chat/completions\n") + write("remote-mapped", "name: remote-mapped\nbackend: cloud-proxy\nproxy:\n mode: translate\n provider: openai\n upstream_url: http://127.0.0.1:1/v1/chat/completions\n upstream_model: big-llm\n") + write("chain-remote", "name: chain-remote\nfailover:\n targets:\n - model: remote\n - model: remote-mapped\n") ss := &system.SystemState{Model: system.Model{ModelsPath: dir}} appConfig := config.NewApplicationConfig() @@ -130,6 +133,33 @@ var _ = Describe("failover chains in the request pipeline", func() { Expect(st.Active).To(Equal("b")) }) + It("sends a remote target its own upstream model, not the chain name", func() { + upstream := map[string]string{} + record := func(c echo.Context) error { + cfg := c.Get(CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) + mu.Lock() + upstream[cfg.Name] = cfg.Proxy.UpstreamModel + mu.Unlock() + return nil + } + behavior["remote"] = func(c echo.Context) error { + _ = record(c) + return errors.New("dial tcp: connection refused") + } + behavior["remote-mapped"] = func(c echo.Context) error { _ = record(c); return served(c) } + Expect(chat("chain-remote").Code).To(Equal(http.StatusOK)) + // The probe checks the same name, so a request cannot fail on a model + // the liveness probe just found. + Expect(upstream).To(Equal(map[string]string{"remote": "remote", "remote-mapped": "big-llm"})) + for _, name := range []string{"remote", "remote-mapped"} { + cfg, ok := re.modelConfigLoader.GetModelConfig(name) + Expect(ok).To(BeTrue()) + Expect(upstream[name]).To(Equal(failover.UpstreamModel(cfg))) + } + stored, _ := re.modelConfigLoader.GetModelConfig("remote") + Expect(stored.Proxy.UpstreamModel).To(BeEmpty(), "the shared config must not change") + }) + It("serves the primary without the failover header", func() { rec := chat("chain") Expect(rec.Header().Get(HeaderServedModel)).To(Equal("a")) diff --git a/core/services/failover/prober.go b/core/services/failover/prober.go index 03108d0c9..c6d412030 100644 --- a/core/services/failover/prober.go +++ b/core/services/failover/prober.go @@ -85,6 +85,17 @@ func UpstreamModel(cfg config.ModelConfig) string { return cfg.Name } +// PrepareTarget readies a copy of a target's config to serve a chain request. +// A remote target gets its upstream model set explicitly: left empty, +// passthrough forwards the client's "model" (the chain name) and translate +// falls back to it, so the upstream would answer 404 for a model the liveness +// probe (which checks UpstreamModel) just found. +func PrepareTarget(cfg *config.ModelConfig) { + if KindOf(*cfg) == KindRemote { + cfg.Proxy.UpstreamModel = UpstreamModel(*cfg) + } +} + func (p *DefaultProber) authorize(req *http.Request, cfg config.ModelConfig) error { key, err := cfg.Proxy.ResolveAPIKey() if err != nil || key == "" { diff --git a/core/services/failover/prober_test.go b/core/services/failover/prober_test.go index 72f03887b..b557c4f96 100644 --- a/core/services/failover/prober_test.go +++ b/core/services/failover/prober_test.go @@ -179,3 +179,20 @@ var _ = Describe("DefaultProber", func() { } }) }) + +var _ = Describe("PrepareTarget", func() { + It("names the upstream model of a remote target the way the probe does", func() { + plain := config.ModelConfig{Name: "argus-llm", Backend: "cloud-proxy"} + PrepareTarget(&plain) + Expect(plain.Proxy.UpstreamModel).To(Equal("argus-llm")) + + mapped := config.ModelConfig{Name: "argus-llm", Backend: "cloud-proxy"} + mapped.Proxy.UpstreamModel = "big-llm" + PrepareTarget(&mapped) + Expect(mapped.Proxy.UpstreamModel).To(Equal("big-llm")) + + local := config.ModelConfig{Name: "gemma", Backend: "llama-cpp"} + PrepareTarget(&local) + Expect(local.Proxy.UpstreamModel).To(BeEmpty()) + }) +}) diff --git a/docs/content/features/model-failover.md b/docs/content/features/model-failover.md index b2ea212b3..c7fa1039a 100644 --- a/docs/content/features/model-failover.md +++ b/docs/content/features/model-failover.md @@ -50,6 +50,9 @@ Rules: - A chain cannot also set `alias` or `backend`. - Responses name the chain as the model. The `X-LocalAI-Served-Model` header names the target that served the request. +- A remote (`cloud-proxy`) target receives its own model name, never the chain + name: `proxy.upstream_model`, or the target name when `upstream_model` is + empty. The health check looks for the same name. ## How the target is chosen diff --git a/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md b/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md index 28d687ca7..57bb7e5b0 100644 --- a/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md +++ b/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md @@ -100,6 +100,12 @@ The behaviour matches aliases. Responses echo the chain name. Usage and traces record `requested=` and `served=` through the existing `ContextKeyRequestedModel` and `ContextKeyServedModel` keys. +The upstream of a remote target never sees the chain name. A request served +through a chain reaches a remote target with that target's upstream model: +`proxy.upstream_model`, or the target name when it is empty. This holds in +passthrough and translate mode, and it is the same name the liveness probe +looks for in `/v1/models` (one helper derives both). + ## Failover manager New package: `core/services/failover`. The application creates one `Manager` at diff --git a/tests/e2e/e2e_failover_test.go b/tests/e2e/e2e_failover_test.go index 67e1227b4..b8e1632d2 100644 --- a/tests/e2e/e2e_failover_test.go +++ b/tests/e2e/e2e_failover_test.go @@ -106,6 +106,11 @@ var _ = Describe("Failover chains", Label("failover"), func() { Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("up-2")) Expect(resp.Header.Get("X-LocalAI-Failover")).To(Equal("fallback")) Expect(chainActive("chain-remote")).To(Equal("up-2")) + // The targets set no upstream_model: each upstream must get its + // target's name (what the liveness probe checks), not the chain + // name the client sent. up-2 is healthy, so only this request + // posted to it. + Expect(upstreamBodyModel(up2)).To(Equal("up-2")) up1.SetScript(chatReply) Eventually(func() string { return chainActive("chain-remote") }, 30*time.Second, 500*time.Millisecond). @@ -116,10 +121,22 @@ var _ = Describe("Failover chains", Label("failover"), func() { Expect(resp2.StatusCode).To(Equal(200)) Expect(resp2.Header.Get("X-LocalAI-Served-Model")).To(Equal("up-1")) Expect(resp2.Header.Get("X-LocalAI-Failover")).To(BeEmpty()) + Expect(upstreamBodyModel(up1)).To(Equal("up-1")) }) }) }) +// upstreamBodyModel returns the "model" field of the last request body the +// fake upstream recorded. +func upstreamBodyModel(up *fakeOpenAIUpstreamServer) string { + _, _, _, body := up.recorder.snapshot() + var req struct { + Model string `json:"model"` + } + Expect(json.Unmarshal(body, &req)).To(Succeed(), string(body)) + return req.Model +} + // chainActive returns the active target of a chain as the REST status reports // it, or "" when the status cannot be read. func chainActive(chain string) string {