From 39b3b311534a50ca0472a4469de8d1feeec014b6 Mon Sep 17 00:00:00 2001 From: mudler-agent Date: Fri, 16 Oct 2026 10:41:45 +0200 Subject: [PATCH] fix(backends): flag a llama-cpp backend that predates native decisions (#12733) A decision request (/v1/systemone, /v1/decisions) sends Score with question_type "systemone". A backend built before native decisions ignores that field and runs plain scoring, so the caller saw "Score was not enabled" (500) or "candidates must be non-empty" (400). The gallery "latest" image can lag behind the server, and the upgrade check compares only tag, version and digest, so /backends/upgrades did not list the backend. Recognise those two errors for a decision request, return 501 with a message that says the backend is older than the server and must be upgraded, and mark the backend stale. The upgrade check now reports a marked backend with a `reason`, and refreshes at once. Reinstalling the backend drops the mark; if the new image is still old, the next request marks it again. Model loading is not affected. Refreshing the "latest" backend image is a separate publishing step and is not part of this change. Refs #12693 Assisted-by: Claude Code:claude-sonnet-5-5 [go vet] [go test] Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/application/startup.go | 3 + core/backend/decision_transport_test.go | 66 +++++++++++++++++++- core/backend/stale_decision_backend.go | 51 +++++++++++++++ core/backend/systemone.go | 2 +- core/gallery/stale_backend.go | 83 +++++++++++++++++++++++++ core/gallery/upgrade.go | 17 +++++ core/gallery/upgrade_test.go | 65 +++++++++++++++++++ docs/content/features/decisions.md | 9 +++ 8 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 core/backend/stale_decision_backend.go create mode 100644 core/gallery/stale_backend.go diff --git a/core/application/startup.go b/core/application/startup.go index c0ddb0f31..08b173213 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -487,6 +487,9 @@ func New(opts ...config.AppOption) (*Application, error) { bmFn := func() galleryop.BackendManager { return application.GalleryService().BackendManager() } uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB(), bmFn) application.upgradeChecker = uc + // A request that proves a backend is older than the server marks it + // stale; refresh the upgrade cache so /backends/upgrades shows it. + gallery.SetStaleBackendNotifier(uc.TriggerCheck) // Refresh the upgrade cache the moment a backend op finishes — otherwise // the UI keeps showing a just-upgraded backend as upgradeable until the // next 6-hour tick. TriggerCheck is non-blocking. diff --git a/core/backend/decision_transport_test.go b/core/backend/decision_transport_test.go index 927f7eee5..54c7d3e92 100644 --- a/core/backend/decision_transport_test.go +++ b/core/backend/decision_transport_test.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/mudler/LocalAI/core/backend" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/trace" grpcPkg "github.com/mudler/LocalAI/pkg/grpc" @@ -16,12 +17,15 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ggrpc "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type nativeDecisionBackend struct { grpcPkg.Backend - request *pb.ScoreRequest - fail bool + request *pb.ScoreRequest + fail bool + failWith error } func (b *nativeDecisionBackend) HealthCheck(context.Context) (bool, error) { return true, nil } @@ -29,6 +33,9 @@ func (b *nativeDecisionBackend) IsBusy() bool { ret func (b *nativeDecisionBackend) Score(_ context.Context, r *pb.ScoreRequest, _ ...ggrpc.CallOption) (*pb.ScoreResponse, error) { b.request = r + if b.failWith != nil { + return nil, b.failWith + } if b.fail { return nil, fmt.Errorf("backend echoed secret-text") } @@ -71,4 +78,59 @@ var _ = Describe("native decision transport", func() { Expect(tr.Error).NotTo(ContainSubstring("secret-text")) } }) + + It("replaces the old-backend scoring error with a plain upgrade message", func() { + for _, msg := range []string{ + "Score was not enabled when the model was loaded; add score to known_usecases", + "candidates must be non-empty", + } { + rec := &nativeDecisionBackend{failWith: status.Error(codes.FailedPrecondition, msg)} + state := &system.SystemState{} + loader := model.NewModelLoader(state) + loader.SetModelRouter(func(_ context.Context, id string, _, _, _, _ string, _ *pb.ModelOptions, _ bool) (*model.Model, error) { + return model.NewModelWithClient(id, "test://stale", rec), nil + }) + cfg := identityModelCfg() + cfg.Backend = "llama-cpp" + cfg.Model = "native-test.gguf" + flags := config.FLAG_DECISIONS + cfg.KnownUsecases = &flags + app := config.NewApplicationConfig(config.WithSystemState(state)) + marked := make(chan struct{}, 1) + gallery.SetStaleBackendNotifier(func() { marked <- struct{}{} }) + DeferCleanup(func() { gallery.SetStaleBackendNotifier(nil) }) + runner := backend.NewDecisionRunner(cfg.Name, func(string) *config.ModelConfig { return &cfg }, loader, app) + req := &schema.SystemOneRequest{State: json.RawMessage(`"x"`), Questions: map[string]schema.SystemOneQuestion{"q": {Type: "noul"}}} + _, err := runner.Decide(context.Background(), req) + Expect(err).To(HaveOccurred()) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + Expect(err.Error()).To(ContainSubstring("older than this server")) + Expect(err.Error()).To(ContainSubstring("llama-cpp")) + Expect(err.Error()).NotTo(ContainSubstring("known_usecases")) + Eventually(marked).Should(Receive()) + } + }) + + It("leaves other backend errors unchanged and does not mark the backend", func() { + rec := &nativeDecisionBackend{failWith: status.Error(codes.InvalidArgument, "bad question")} + state := &system.SystemState{} + loader := model.NewModelLoader(state) + loader.SetModelRouter(func(_ context.Context, id string, _, _, _, _ string, _ *pb.ModelOptions, _ bool) (*model.Model, error) { + return model.NewModelWithClient(id, "test://other", rec), nil + }) + cfg := identityModelCfg() + cfg.Backend = "vllm-cpp" + cfg.Model = "native-test.gguf" + flags := config.FLAG_DECISIONS + cfg.KnownUsecases = &flags + app := config.NewApplicationConfig(config.WithSystemState(state)) + notified := false + gallery.SetStaleBackendNotifier(func() { notified = true }) + DeferCleanup(func() { gallery.SetStaleBackendNotifier(nil) }) + runner := backend.NewDecisionRunner(cfg.Name, func(string) *config.ModelConfig { return &cfg }, loader, app) + req := &schema.SystemOneRequest{State: json.RawMessage(`"x"`), Questions: map[string]schema.SystemOneQuestion{"q": {Type: "noul"}}} + _, err := runner.Decide(context.Background(), req) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + Expect(notified).To(BeFalse()) + }) }) diff --git a/core/backend/stale_decision_backend.go b/core/backend/stale_decision_backend.go new file mode 100644 index 000000000..c9bfa911f --- /dev/null +++ b/core/backend/stale_decision_backend.go @@ -0,0 +1,51 @@ +package backend + +import ( + "strings" + + "github.com/mudler/LocalAI/core/gallery" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// staleDecisionMessage is returned in place of the raw backend error when a +// backend answers a decision request as a plain Score call. Constant text on +// purpose: it never echoes backend or request content. +const staleDecisionMessage = "the backend is older than this server and has no native decision support; " + + "upgrade the %q backend (see /backends/upgrades), then retry" + +// A decision request is a Score call with question_type "systemone". A backend +// that knows the field never returns the two errors below for it: it either +// runs the decision or answers UNIMPLEMENTED. A backend built before native +// decisions ignores the unknown field and runs plain scoring, which fails with +// one of these messages. So they identify an old backend, not a bad model +// configuration, and adding "score" to known_usecases is not the fix. +var staleDecisionSignatures = []string{ + "Score was not enabled when the model was loaded", + "candidates must be non-empty", +} + +// staleDecisionBackendError reports whether err, returned for a decision +// request, comes from a backend that predates native decisions. +func staleDecisionBackendError(err error) bool { + if err == nil { + return false + } + msg := status.Convert(err).Message() + for _, sig := range staleDecisionSignatures { + if strings.Contains(msg, sig) { + return true + } + } + return false +} + +// handleStaleDecisionBackend turns the old-backend failure into a plain +// error and flags the backend in the upgrade check. Other errors pass through. +func handleStaleDecisionBackend(backendName string, err error) error { + if !staleDecisionBackendError(err) { + return err + } + gallery.MarkBackendStale(backendName, "backend does not support native decisions (older than this server)") + return status.Errorf(codes.Unimplemented, staleDecisionMessage, backendName) +} diff --git a/core/backend/systemone.go b/core/backend/systemone.go index 471eb2372..f4f356ce3 100644 --- a/core/backend/systemone.go +++ b/core/backend/systemone.go @@ -72,7 +72,7 @@ func ModelSystemOne(requestJSON string, loader *model.ModelLoader, modelConfig c }) } if err != nil { - return "", err + return "", handleStaleDecisionBackend(modelConfig.Backend, err) } return resp.GetResponseJson(), nil }, nil diff --git a/core/gallery/stale_backend.go b/core/gallery/stale_backend.go new file mode 100644 index 000000000..6a176ad4b --- /dev/null +++ b/core/gallery/stale_backend.go @@ -0,0 +1,83 @@ +package gallery + +import ( + "sync" + "time" +) + +// A backend can be current by tag, version and digest and still be too old +// for the server. The gallery "latest" image of a backend can lag behind the +// server code, so a tag or digest comparison reports "up to date" while the +// backend lacks a feature the server relies on. The server cannot ask an +// unloaded backend for its features, so it learns this when a request proves +// it: the request carries a field only a new backend understands, and an old +// backend answers as if the field were absent. +// +// MarkBackendStale records such an observation. CheckUpgradesAgainst then +// reports every backend installed before that moment as upgradeable, with the +// reason. Reinstalling the backend (which sets a newer InstalledAt) drops the +// mark, so no explicit clearing is needed. If the reinstalled image is still +// old, the next failing request marks it again. +// +// The marks are per process. In distributed mode each frontend learns from the +// requests it serves. + +type staleMark struct { + at time.Time + reason string +} + +var ( + staleMu sync.Mutex + staleBackends = map[string]staleMark{} + staleNotify func() +) + +// MarkBackendStale records that the backend configured as name (a concrete +// backend name or its alias, for example "llama-cpp") failed a request in a +// way that shows it is older than the server. reason is shown to the operator. +func MarkBackendStale(name, reason string) { + if name == "" { + return + } + staleMu.Lock() + staleBackends[name] = staleMark{at: time.Now(), reason: reason} + notify := staleNotify + staleMu.Unlock() + if notify != nil { + notify() + } +} + +// SetStaleBackendNotifier registers a function called after each +// MarkBackendStale, so the upgrade cache can refresh at once instead of at +// the next periodic check. The function must not block. +func SetStaleBackendNotifier(fn func()) { + staleMu.Lock() + staleNotify = fn + staleMu.Unlock() +} + +// staleReason returns why the installed backend was marked stale, if a mark +// matches its name or alias and was recorded after it was installed. +func staleReason(md *BackendMetadata) (string, bool) { + if md == nil { + return "", false + } + installedAt, err := time.Parse(time.RFC3339, md.InstalledAt) + if err != nil { + // Unknown install time: treat the backend as installed before the mark. + installedAt = time.Time{} + } + staleMu.Lock() + defer staleMu.Unlock() + for _, key := range []string{md.Name, md.Alias} { + if key == "" { + continue + } + if m, ok := staleBackends[key]; ok && m.at.After(installedAt) { + return m.reason, true + } + } + return "", false +} diff --git a/core/gallery/upgrade.go b/core/gallery/upgrade.go index 162ef533f..09a548e37 100644 --- a/core/gallery/upgrade.go +++ b/core/gallery/upgrade.go @@ -23,6 +23,10 @@ type UpgradeInfo struct { AvailableVersion string `json:"available_version"` InstalledDigest string `json:"installed_digest,omitempty"` AvailableDigest string `json:"available_digest,omitempty"` + // Reason says why the backend is flagged when version and digest alone + // would not flag it: the backend failed a request in a way that shows it + // is older than the server. Empty for ordinary version or digest upgrades. + Reason string `json:"reason,omitempty"` // NodeDrift lists nodes whose installed version or digest differs from // the cluster majority. Non-empty means the cluster has diverged and an // upgrade will realign it. Empty in single-node mode. @@ -129,6 +133,19 @@ func CheckUpgradesAgainst(ctx context.Context, galleries []config.Gallery, syste installedDigest := installed.Metadata.Digest galleryVersion := galleryEntry.Version + // A request showed this backend is older than the server, even though + // its tag, version or digest matches the gallery. See stale_backend.go. + if reason, stale := staleReason(installed.Metadata); stale { + result[installed.Metadata.Name] = UpgradeInfo{ + BackendName: installed.Metadata.Name, + InstalledVersion: installedVersion, + AvailableVersion: galleryVersion, + InstalledDigest: installedDigest, + Reason: reason, + } + continue + } + // Detect cluster drift: does every node report the same version+digest? // In single-node mode this stays empty (Nodes is nil). majority, drift := summarizeNodeDrift(installed.Nodes) diff --git a/core/gallery/upgrade_test.go b/core/gallery/upgrade_test.go index ae1be8fcc..2320457d9 100644 --- a/core/gallery/upgrade_test.go +++ b/core/gallery/upgrade_test.go @@ -144,6 +144,71 @@ var _ = Describe("Upgrade Detection and Execution", func() { }) }) + Describe("stale backend marks", func() { + // Backend names are unique per test: the marks live in the process. + installAliased := func(name, alias, installedAt string) { + dir := filepath.Join(backendsPath, name) + Expect(os.MkdirAll(dir, 0750)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755)).To(Succeed()) + data, err := json.Marshal(BackendMetadata{Name: name, Alias: alias, InstalledAt: installedAt}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + writeGalleryYAML([]GalleryBackend{{ + Metadata: Metadata{Name: name}, + URI: filepath.Join(tempDir, "some-source"), + }}) + } + + It("flags a backend that is current by tag once a request proves it is old", func() { + installAliased("stale-cpp", "stale-alias", time.Now().Add(-time.Hour).Format(time.RFC3339)) + + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(BeEmpty()) + + MarkBackendStale("stale-alias", "no native decisions") + upgrades, err = CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(HaveKey("stale-cpp")) + Expect(upgrades["stale-cpp"].Reason).To(Equal("no native decisions")) + }) + + It("matches a mark on the concrete backend name", func() { + installAliased("stale-named", "", time.Now().Add(-time.Hour).Format(time.RFC3339)) + MarkBackendStale("stale-named", "old") + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(HaveKey("stale-named")) + }) + + It("drops the mark once the backend is reinstalled after it", func() { + installAliased("stale-fresh", "stale-fresh-alias", time.Now().Add(-time.Hour).Format(time.RFC3339)) + MarkBackendStale("stale-fresh-alias", "old") + installAliased("stale-fresh", "stale-fresh-alias", time.Now().Add(time.Minute).Format(time.RFC3339)) + + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(BeEmpty()) + }) + + It("ignores marks for other backends and empty names", func() { + installAliased("stale-other", "", time.Now().Add(-time.Hour).Format(time.RFC3339)) + MarkBackendStale("", "ignored") + MarkBackendStale("not-installed", "ignored") + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(BeEmpty()) + }) + + It("calls the notifier on each mark", func() { + calls := 0 + SetStaleBackendNotifier(func() { calls++ }) + DeferCleanup(func() { SetStaleBackendNotifier(nil) }) + MarkBackendStale("stale-notify", "old") + Expect(calls).To(Equal(1)) + }) + }) + // CheckUpgradesAgainst is the entry point used by DistributedBackendManager. // It takes installed backends directly — typically aggregated from workers — // instead of reading the frontend filesystem. These tests exercise drift diff --git a/docs/content/features/decisions.md b/docs/content/features/decisions.md index 94d8096bf..9cfcf4b2e 100644 --- a/docs/content/features/decisions.md +++ b/docs/content/features/decisions.md @@ -290,6 +290,15 @@ accept 2–10 levels; this backend-specific limit does not constrain vllm-cpp. Older forks without native decision support return 501. Missing decision metadata also returns 501, while backend invalid requests return 400. +A `llama-cpp` backend image built before native decisions cannot answer these +requests, even when the gallery reports it as up to date. LocalAI detects this +from the failed request: it returns 501 with a message that the backend is older +than the server, and `GET /backends/upgrades` lists the backend with a `reason` +field. Upgrade the backend, or install a current build such as the +`-development` variant. Adding `score` to `known_usecases` does not help. The +flag clears when the backend is reinstalled; if the reinstalled image is still +old, the next decision request flags it again. + ### Bounded image input The public routes and internal decision validator share an image contract. Supply