diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index 3e53a81fd..8cf4e4050 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -419,6 +419,39 @@ func DefaultRegistry() map[string]FieldMetaOverride { Order: 0, }, + // --- Failover --- + "failover.targets": { + Section: "failover", + Label: "Failover targets", + Description: "Ordered list of models that serve this chain. The first healthy target serves each request; later targets take over when it fails. Mark a local target warm to keep it loaded.", + Component: "json-editor", + Order: 0, + }, + "failover.probe.interval": { + Section: "failover", Label: "Probe interval", Component: "input", Order: 1, Advanced: true, + Description: "How often an idle target is checked, as a duration (default 15s).", Placeholder: "15s", + }, + "failover.probe.timeout": { + Section: "failover", Label: "Probe timeout", Component: "input", Order: 2, Advanced: true, + Description: "How long one probe may take (default 5s).", Placeholder: "5s", + }, + "failover.trip.errors": { + Section: "failover", Label: "Errors to trip", Component: "number", Order: 3, Advanced: true, + Description: "Failures within the trip window that mark a target down (default 1).", + }, + "failover.trip.window": { + Section: "failover", Label: "Trip window", Component: "input", Order: 4, Advanced: true, + Description: "Window in which failures are counted (default 30s).", Placeholder: "30s", + }, + "failover.recovery.probes": { + Section: "failover", Label: "Recovery probes", Component: "number", Order: 5, Advanced: true, + Description: "Consecutive real test requests a target must pass before it is used again (default 3).", + }, + "failover.recovery.min_dwell": { + Section: "failover", Label: "Minimum time on fallback", Component: "input", Order: 6, Advanced: true, + Description: "Minimum time on a lower target before traffic moves back to a recovered higher one (default 60s).", Placeholder: "60s", + }, + // --- Pipeline --- "pipeline.llm": { Section: "pipeline", diff --git a/core/config/meta/registry_test.go b/core/config/meta/registry_test.go index c3f1a494d..eff0cbf58 100644 --- a/core/config/meta/registry_test.go +++ b/core/config/meta/registry_test.go @@ -28,6 +28,17 @@ var _ = Describe("alias field metadata", func() { } Expect(found).To(BeTrue(), "DefaultSections should include an alias section") }) + + It("registers the failover section", func() { + reg := meta.DefaultRegistry() + Expect(reg).To(HaveKey("failover.targets")) + Expect(reg["failover.targets"].Section).To(Equal("failover")) + var ids []string + for _, s := range meta.DefaultSections() { + ids = append(ids, s.ID) + } + Expect(ids).To(ContainElement("failover")) + }) }) var _ = Describe("MCP field metadata", func() { diff --git a/core/config/meta/types.go b/core/config/meta/types.go index a29e66967..b7774d157 100644 --- a/core/config/meta/types.go +++ b/core/config/meta/types.go @@ -70,6 +70,7 @@ func DefaultSections() []Section { return []Section{ {ID: "general", Label: "General", Icon: "settings", Order: 0}, {ID: "alias", Label: "Alias", Icon: "git-merge", Order: 5}, + {ID: "failover", Label: "Failover", Icon: "git-merge", Order: 6}, {ID: "llm", Label: "LLM", Icon: "cpu", Order: 10}, {ID: "parameters", Label: "Parameters", Icon: "sliders", Order: 20}, {ID: "templates", Label: "Templates", Icon: "file-text", Order: 30}, diff --git a/core/config/model_config.go b/core/config/model_config.go index 9754ef6ac..33fe12d21 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -75,6 +75,10 @@ type ModelConfig struct { // at create/swap time). See docs/content for Model Aliases. Alias string `yaml:"alias,omitempty" json:"alias,omitempty"` + // Failover makes this config a failover chain over other models. Like an + // alias it has no backend of its own. + Failover *FailoverConfig `yaml:"failover,omitempty" json:"failover,omitempty"` + F16 *bool `yaml:"f16,omitempty" json:"f16,omitempty"` Threads *int `yaml:"threads,omitempty" json:"threads,omitempty"` Debug *bool `yaml:"debug,omitempty" json:"debug,omitempty"` @@ -1644,6 +1648,13 @@ func (c *ModelConfig) Validate() (bool, error) { return false, fmt.Errorf("a config with artifacts must declare exactly one %q target, found %d", modelartifacts.TargetModel, primaries) } + if c.IsFailover() { + if err := c.validateFailover(); err != nil { + return false, err + } + return true, nil + } + // An alias is a pure redirect: validate only its own shape here. Target // existence and the no-chain rule need the full config set, so the loader // (load-time) and the create/swap endpoints enforce those. diff --git a/core/config/model_config_failover.go b/core/config/model_config_failover.go new file mode 100644 index 000000000..e33a51ea8 --- /dev/null +++ b/core/config/model_config_failover.go @@ -0,0 +1,150 @@ +package config + +import ( + "fmt" + "time" +) + +// FailoverConfig turns a model config into a failover chain: requests for the +// chain name are served by its highest-priority healthy target. See +// core/services/failover for the runtime side. +type FailoverConfig struct { + Targets []FailoverTarget `yaml:"targets" json:"targets"` + Probe FailoverProbe `yaml:"probe,omitempty" json:"probe,omitempty"` + Trip FailoverTrip `yaml:"trip,omitempty" json:"trip,omitempty"` + Recovery FailoverRecovery `yaml:"recovery,omitempty" json:"recovery,omitempty"` +} + +type FailoverTarget struct { + Model string `yaml:"model" json:"model"` + // Warm keeps a local target loaded and exempt from eviction, so a switch + // does not wait for a cold load. + Warm bool `yaml:"warm,omitempty" json:"warm,omitempty"` +} + +type FailoverProbe struct { + Interval string `yaml:"interval,omitempty" json:"interval,omitempty"` + Timeout string `yaml:"timeout,omitempty" json:"timeout,omitempty"` +} + +type FailoverTrip struct { + Errors int `yaml:"errors,omitempty" json:"errors,omitempty"` + Window string `yaml:"window,omitempty" json:"window,omitempty"` +} + +type FailoverRecovery struct { + Probes int `yaml:"probes,omitempty" json:"probes,omitempty"` + MinDwell string `yaml:"min_dwell,omitempty" json:"min_dwell,omitempty"` +} + +const ( + DefaultFailoverProbeInterval = 15 * time.Second + DefaultFailoverProbeTimeout = 5 * time.Second + DefaultFailoverTripErrors = 1 + DefaultFailoverTripWindow = 30 * time.Second + DefaultFailoverRecoveryProbes = 3 + DefaultFailoverMinDwell = 60 * time.Second +) + +// IsFailover reports whether this config is a failover chain. +func (c ModelConfig) IsFailover() bool { return c.Failover != nil } + +func (f FailoverConfig) ProbeInterval() time.Duration { + return durationOr(f.Probe.Interval, DefaultFailoverProbeInterval) +} +func (f FailoverConfig) ProbeTimeout() time.Duration { + return durationOr(f.Probe.Timeout, DefaultFailoverProbeTimeout) +} +func (f FailoverConfig) TripWindow() time.Duration { + return durationOr(f.Trip.Window, DefaultFailoverTripWindow) +} +func (f FailoverConfig) MinDwell() time.Duration { + return durationOr(f.Recovery.MinDwell, DefaultFailoverMinDwell) +} +func (f FailoverConfig) TripErrors() int { + if f.Trip.Errors <= 0 { + return DefaultFailoverTripErrors + } + return f.Trip.Errors +} +func (f FailoverConfig) RecoveryProbes() int { + if f.Recovery.Probes <= 0 { + return DefaultFailoverRecoveryProbes + } + return f.Recovery.Probes +} + +// WarmFailoverTargets returns the targets marked warm, in chain order. +func (c ModelConfig) WarmFailoverTargets() []string { + if c.Failover == nil { + return nil + } + var out []string + for _, t := range c.Failover.Targets { + if t.Warm { + out = append(out, t.Model) + } + } + return out +} + +func durationOr(s string, def time.Duration) time.Duration { + if s == "" { + return def + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return def + } + return d +} + +// validateFailover checks what a chain can check without other configs. +// Target existence is checked by ModelConfigLoader.ValidateFailoverTargets. +func (c *ModelConfig) validateFailover() error { + if c.Name == "" { + return fmt.Errorf("failover config requires a name") + } + if c.IsAlias() { + return fmt.Errorf("model %q cannot set both alias and failover", c.Name) + } + if c.Backend != "" || c.Model != "" { + return fmt.Errorf("failover config %q must not set backend or parameters.model: a chain is a pure redirect", c.Name) + } + f := c.Failover + if len(f.Targets) < 2 { + return fmt.Errorf("failover chain %q needs at least 2 targets", c.Name) + } + seen := map[string]bool{} + for _, t := range f.Targets { + switch { + case t.Model == "": + return fmt.Errorf("failover chain %q has a target with no model", c.Name) + case t.Model == c.Name: + return fmt.Errorf("failover chain %q cannot list itself", c.Name) + case seen[t.Model]: + return fmt.Errorf("failover chain %q lists %q twice", c.Name, t.Model) + } + seen[t.Model] = true + } + for key, v := range map[string]string{ + "probe.interval": f.Probe.Interval, + "probe.timeout": f.Probe.Timeout, + "trip.window": f.Trip.Window, + "recovery.min_dwell": f.Recovery.MinDwell, + } { + if v == "" { + continue + } + if d, err := time.ParseDuration(v); err != nil || d <= 0 { + return fmt.Errorf("failover chain %q: invalid %s %q", c.Name, key, v) + } + } + if f.Trip.Errors < 0 { + return fmt.Errorf("failover chain %q: trip.errors must not be negative", c.Name) + } + if f.Recovery.Probes < 0 { + return fmt.Errorf("failover chain %q: recovery.probes must not be negative", c.Name) + } + return nil +} diff --git a/core/config/model_config_failover_test.go b/core/config/model_config_failover_test.go new file mode 100644 index 000000000..e79ed5f45 --- /dev/null +++ b/core/config/model_config_failover_test.go @@ -0,0 +1,68 @@ +package config + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +var _ = Describe("ModelConfig failover", func() { + chain := func(targets ...string) ModelConfig { + c := ModelConfig{Name: "chain", Failover: &FailoverConfig{}} + for _, t := range targets { + c.Failover.Targets = append(c.Failover.Targets, FailoverTarget{Model: t}) + } + return c + } + + It("parses the YAML block and applies defaults", func() { + var c ModelConfig + Expect(yaml.Unmarshal([]byte(` +name: assistant-llm +failover: + targets: + - model: argus-llm + - model: gemma-local + warm: true + recovery: + probes: 5 +`), &c)).To(Succeed()) + Expect(c.IsFailover()).To(BeTrue()) + Expect(c.Failover.Targets).To(Equal([]FailoverTarget{{Model: "argus-llm"}, {Model: "gemma-local", Warm: true}})) + Expect(c.Failover.ProbeInterval()).To(Equal(15 * time.Second)) + Expect(c.Failover.ProbeTimeout()).To(Equal(5 * time.Second)) + Expect(c.Failover.TripErrors()).To(Equal(1)) + Expect(c.Failover.TripWindow()).To(Equal(30 * time.Second)) + Expect(c.Failover.RecoveryProbes()).To(Equal(5)) + Expect(c.Failover.MinDwell()).To(Equal(60 * time.Second)) + Expect(c.WarmFailoverTargets()).To(Equal([]string{"gemma-local"})) + }) + + It("accepts a valid chain", func() { + c := chain("a", "b") + ok, err := c.Validate() + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeTrue()) + }) + + DescribeTable("rejects invalid chains", + func(mutate func(*ModelConfig), want string) { + c := chain("a", "b") + mutate(&c) + ok, err := c.Validate() + Expect(ok).To(BeFalse()) + Expect(err).To(MatchError(ContainSubstring(want))) + }, + Entry("alias and failover", func(c *ModelConfig) { c.Alias = "x" }, "both alias and failover"), + Entry("backend set", func(c *ModelConfig) { c.Backend = "llama-cpp" }, "must not set backend"), + Entry("one target", func(c *ModelConfig) { c.Failover.Targets = c.Failover.Targets[:1] }, "at least 2 targets"), + Entry("empty target", func(c *ModelConfig) { c.Failover.Targets[1].Model = "" }, "no model"), + Entry("self target", func(c *ModelConfig) { c.Failover.Targets[1].Model = "chain" }, "cannot list itself"), + Entry("duplicate target", func(c *ModelConfig) { c.Failover.Targets[1].Model = "a" }, "twice"), + Entry("bad duration", func(c *ModelConfig) { c.Failover.Probe.Interval = "soon" }, "invalid probe.interval"), + Entry("negative errors", func(c *ModelConfig) { c.Failover.Trip.Errors = -1 }, "trip.errors"), + Entry("no name", func(c *ModelConfig) { c.Name = "" }, "requires a name"), + ) +})