diff --git a/core/http/endpoints/openai/realtime_model.go b/core/http/endpoints/openai/realtime_model.go index a90a0e4e5..ee42bde2d 100644 --- a/core/http/endpoints/openai/realtime_model.go +++ b/core/http/endpoints/openai/realtime_model.go @@ -1032,6 +1032,14 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model // takes the chain's active target, so everything that inspects stage // configs at session start (voice, reasoning, templates) sees a real model. stageChains := map[string]string{} + loadTarget := func(name string) (*config.ModelConfig, error) { + cfg, err := cl.LoadResolvedModelConfig(name, ml.ModelPath, appConfig.ToConfigLoaderOptions()...) + if err != nil { + return nil, err + } + failover.PrepareTarget(cfg) + return cfg, nil + } resolveStage := func(stage string, cfg *config.ModelConfig) (*config.ModelConfig, error) { if cfg == nil || !cfg.IsFailover() { return cfg, nil @@ -1044,7 +1052,7 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model return nil, fmt.Errorf("failover chain %q not found", cfg.Name) } stageChains[stage] = cfg.Name - return cl.LoadResolvedModelConfig(st.Active, ml.ModelPath, appConfig.ToConfigLoaderOptions()...) + return loadTarget(st.Active) } cfgVAD, err := cl.LoadResolvedModelConfig(pipeline.VAD, ml.ModelPath, appConfig.ToConfigLoaderOptions()...) @@ -1185,12 +1193,10 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model appConfig: appConfig, evaluator: evaluator, - stageChains: stageChains, - stageTargetConfig: func(name string) (*config.ModelConfig, error) { - return cl.LoadResolvedModelConfig(name, ml.ModelPath, appConfig.ToConfigLoaderOptions()...) - }, - tuneLLM: tuneLLM, - appTracing: appConfig.EnableTracing, + stageChains: stageChains, + stageTargetConfig: loadTarget, + tuneLLM: tuneLLM, + appTracing: appConfig.EnableTracing, } if routing != nil { wm.routerDeps = routing.Deps diff --git a/core/http/middleware/failover.go b/core/http/middleware/failover.go index f3744fdd9..853a3f22a 100644 --- a/core/http/middleware/failover.go +++ b/core/http/middleware/failover.go @@ -58,6 +58,7 @@ func (re *RequestExtractor) resolveFailover(c echo.Context, requested string, ch return nil, fmt.Errorf("failover chain %q: target %q is disabled", chain.Name, cfg.Name) } if err == nil { + failover.PrepareTarget(cfg) // cfg is a copy c.Set(ContextKeyRequestedModel, requested) c.Set(ContextKeyServedModel, cfg.Name) setFailoverHeaders(c.Response().Header(), st.attempt) diff --git a/core/http/middleware/failover_test.go b/core/http/middleware/failover_test.go index 81fff5e02..b5e8ba5b7 100644 --- a/core/http/middleware/failover_test.go +++ b/core/http/middleware/failover_test.go @@ -79,6 +79,9 @@ var _ = Describe("failover chains in the request pipeline", func() { write("off", "name: off\nbackend: fake-o\ndisabled: true\n") write("chain-capped", "name: chain-capped\nfailover:\n targets:\n - model: capped\n - model: b\n") write("chain-off", "name: chain-off\nfailover:\n targets:\n - model: off\n - model: b\n") + write("remote", "name: remote\nbackend: cloud-proxy\nproxy:\n mode: passthrough\n upstream_url: http://127.0.0.1:1/v1/chat/completions\n") + write("remote-mapped", "name: remote-mapped\nbackend: cloud-proxy\nproxy:\n mode: translate\n provider: openai\n upstream_url: http://127.0.0.1:1/v1/chat/completions\n upstream_model: big-llm\n") + write("chain-remote", "name: chain-remote\nfailover:\n targets:\n - model: remote\n - model: remote-mapped\n") ss := &system.SystemState{Model: system.Model{ModelsPath: dir}} appConfig := config.NewApplicationConfig() @@ -130,6 +133,33 @@ var _ = Describe("failover chains in the request pipeline", func() { Expect(st.Active).To(Equal("b")) }) + It("sends a remote target its own upstream model, not the chain name", func() { + upstream := map[string]string{} + record := func(c echo.Context) error { + cfg := c.Get(CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) + mu.Lock() + upstream[cfg.Name] = cfg.Proxy.UpstreamModel + mu.Unlock() + return nil + } + behavior["remote"] = func(c echo.Context) error { + _ = record(c) + return errors.New("dial tcp: connection refused") + } + behavior["remote-mapped"] = func(c echo.Context) error { _ = record(c); return served(c) } + Expect(chat("chain-remote").Code).To(Equal(http.StatusOK)) + // The probe checks the same name, so a request cannot fail on a model + // the liveness probe just found. + Expect(upstream).To(Equal(map[string]string{"remote": "remote", "remote-mapped": "big-llm"})) + for _, name := range []string{"remote", "remote-mapped"} { + cfg, ok := re.modelConfigLoader.GetModelConfig(name) + Expect(ok).To(BeTrue()) + Expect(upstream[name]).To(Equal(failover.UpstreamModel(cfg))) + } + stored, _ := re.modelConfigLoader.GetModelConfig("remote") + Expect(stored.Proxy.UpstreamModel).To(BeEmpty(), "the shared config must not change") + }) + It("serves the primary without the failover header", func() { rec := chat("chain") Expect(rec.Header().Get(HeaderServedModel)).To(Equal("a")) diff --git a/core/services/failover/prober.go b/core/services/failover/prober.go index 03108d0c9..c6d412030 100644 --- a/core/services/failover/prober.go +++ b/core/services/failover/prober.go @@ -85,6 +85,17 @@ func UpstreamModel(cfg config.ModelConfig) string { return cfg.Name } +// PrepareTarget readies a copy of a target's config to serve a chain request. +// A remote target gets its upstream model set explicitly: left empty, +// passthrough forwards the client's "model" (the chain name) and translate +// falls back to it, so the upstream would answer 404 for a model the liveness +// probe (which checks UpstreamModel) just found. +func PrepareTarget(cfg *config.ModelConfig) { + if KindOf(*cfg) == KindRemote { + cfg.Proxy.UpstreamModel = UpstreamModel(*cfg) + } +} + func (p *DefaultProber) authorize(req *http.Request, cfg config.ModelConfig) error { key, err := cfg.Proxy.ResolveAPIKey() if err != nil || key == "" { diff --git a/core/services/failover/prober_test.go b/core/services/failover/prober_test.go index 72f03887b..b557c4f96 100644 --- a/core/services/failover/prober_test.go +++ b/core/services/failover/prober_test.go @@ -179,3 +179,20 @@ var _ = Describe("DefaultProber", func() { } }) }) + +var _ = Describe("PrepareTarget", func() { + It("names the upstream model of a remote target the way the probe does", func() { + plain := config.ModelConfig{Name: "argus-llm", Backend: "cloud-proxy"} + PrepareTarget(&plain) + Expect(plain.Proxy.UpstreamModel).To(Equal("argus-llm")) + + mapped := config.ModelConfig{Name: "argus-llm", Backend: "cloud-proxy"} + mapped.Proxy.UpstreamModel = "big-llm" + PrepareTarget(&mapped) + Expect(mapped.Proxy.UpstreamModel).To(Equal("big-llm")) + + local := config.ModelConfig{Name: "gemma", Backend: "llama-cpp"} + PrepareTarget(&local) + Expect(local.Proxy.UpstreamModel).To(BeEmpty()) + }) +}) diff --git a/docs/content/features/model-failover.md b/docs/content/features/model-failover.md index b2ea212b3..c7fa1039a 100644 --- a/docs/content/features/model-failover.md +++ b/docs/content/features/model-failover.md @@ -50,6 +50,9 @@ Rules: - A chain cannot also set `alias` or `backend`. - Responses name the chain as the model. The `X-LocalAI-Served-Model` header names the target that served the request. +- A remote (`cloud-proxy`) target receives its own model name, never the chain + name: `proxy.upstream_model`, or the target name when `upstream_model` is + empty. The health check looks for the same name. ## How the target is chosen diff --git a/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md b/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md index 28d687ca7..57bb7e5b0 100644 --- a/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md +++ b/docs/superpowers/specs/2026-09-26-model-failover-chains-design.md @@ -100,6 +100,12 @@ The behaviour matches aliases. Responses echo the chain name. Usage and traces record `requested=` and `served=` through the existing `ContextKeyRequestedModel` and `ContextKeyServedModel` keys. +The upstream of a remote target never sees the chain name. A request served +through a chain reaches a remote target with that target's upstream model: +`proxy.upstream_model`, or the target name when it is empty. This holds in +passthrough and translate mode, and it is the same name the liveness probe +looks for in `/v1/models` (one helper derives both). + ## Failover manager New package: `core/services/failover`. The application creates one `Manager` at diff --git a/tests/e2e/e2e_failover_test.go b/tests/e2e/e2e_failover_test.go index 67e1227b4..b8e1632d2 100644 --- a/tests/e2e/e2e_failover_test.go +++ b/tests/e2e/e2e_failover_test.go @@ -106,6 +106,11 @@ var _ = Describe("Failover chains", Label("failover"), func() { Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("up-2")) Expect(resp.Header.Get("X-LocalAI-Failover")).To(Equal("fallback")) Expect(chainActive("chain-remote")).To(Equal("up-2")) + // The targets set no upstream_model: each upstream must get its + // target's name (what the liveness probe checks), not the chain + // name the client sent. up-2 is healthy, so only this request + // posted to it. + Expect(upstreamBodyModel(up2)).To(Equal("up-2")) up1.SetScript(chatReply) Eventually(func() string { return chainActive("chain-remote") }, 30*time.Second, 500*time.Millisecond). @@ -116,10 +121,22 @@ var _ = Describe("Failover chains", Label("failover"), func() { Expect(resp2.StatusCode).To(Equal(200)) Expect(resp2.Header.Get("X-LocalAI-Served-Model")).To(Equal("up-1")) Expect(resp2.Header.Get("X-LocalAI-Failover")).To(BeEmpty()) + Expect(upstreamBodyModel(up1)).To(Equal("up-1")) }) }) }) +// upstreamBodyModel returns the "model" field of the last request body the +// fake upstream recorded. +func upstreamBodyModel(up *fakeOpenAIUpstreamServer) string { + _, _, _, body := up.recorder.snapshot() + var req struct { + Model string `json:"model"` + } + Expect(json.Unmarshal(body, &req)).To(Succeed(), string(body)) + return req.Model +} + // chainActive returns the active target of a chain as the REST status reports // it, or "" when the status cannot be read. func chainActive(chain string) string {