fix(backends): flag a llama-cpp backend that predates native decisions (#12733)

A decision request (/v1/systemone, /v1/decisions) sends Score with question_type "systemone". A backend built before native decisions ignores that field and runs plain scoring, so the caller saw "Score was not enabled" (500) or "candidates must be non-empty" (400). The gallery "latest" image can lag behind the server, and the upgrade check compares only tag, version and digest, so /backends/upgrades did not list the backend. Recognise those two errors for a decision request, return 501 with a message that says the backend is older than the server and must be upgraded, and mark the backend stale. The upgrade check now reports a marked backend with a `reason`, and refreshes at once. Reinstalling the backend drops the mark; if the new image is still old, the next request marks it again. Model loading is not affected. Refreshing the "latest" backend image is a separate publishing step and is not part of this change. Refs #12693 Assisted-by: Claude Code:claude-sonnet-5-5 [go vet] [go test] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
2026-10-28 09:08:54 -04:00 · 2026-10-16 10:41:45 +02:00
1 parent 62fa19b833
commit 39b3b31153
@@ -487,6 +487,9 @@ func New(opts ...config.AppOption) (*Application, error) {
 		bmFn := func() galleryop.BackendManager { return application.GalleryService().BackendManager() }
 		uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB(), bmFn)
 		application.upgradeChecker = uc
+		// A request that proves a backend is older than the server marks it
+		// stale; refresh the upgrade cache so /backends/upgrades shows it.
+		gallery.SetStaleBackendNotifier(uc.TriggerCheck)
 		// Refresh the upgrade cache the moment a backend op finishes — otherwise
 		// the UI keeps showing a just-upgraded backend as upgradeable until the
 		// next 6-hour tick. TriggerCheck is non-blocking.
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"github.com/mudler/LocalAI/core/backend"
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/mudler/LocalAI/core/trace"
 	grpcPkg "github.com/mudler/LocalAI/pkg/grpc"
@@ -16,12 +17,15 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	ggrpc "google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )

 type nativeDecisionBackend struct {
 	grpcPkg.Backend
-	request *pb.ScoreRequest
-	fail    bool
+	request  *pb.ScoreRequest
+	fail     bool
+	failWith error
 }

 func (b *nativeDecisionBackend) HealthCheck(context.Context) (bool, error) { return true, nil }
@@ -29,6 +33,9 @@ func (b *nativeDecisionBackend) IsBusy() bool                              { ret

 func (b *nativeDecisionBackend) Score(_ context.Context, r *pb.ScoreRequest, _ ...ggrpc.CallOption) (*pb.ScoreResponse, error) {
 	b.request = r
+	if b.failWith != nil {
+		return nil, b.failWith
+	}
 	if b.fail {
 		return nil, fmt.Errorf("backend echoed secret-text")
 	}
@@ -71,4 +78,59 @@ var _ = Describe("native decision transport", func() {
 			Expect(tr.Error).NotTo(ContainSubstring("secret-text"))
 		}
 	})
+
+	It("replaces the old-backend scoring error with a plain upgrade message", func() {
+		for _, msg := range []string{
+			"Score was not enabled when the model was loaded; add score to known_usecases",
+			"candidates must be non-empty",
+		} {
+			rec := &nativeDecisionBackend{failWith: status.Error(codes.FailedPrecondition, msg)}
+			state := &system.SystemState{}
+			loader := model.NewModelLoader(state)
+			loader.SetModelRouter(func(_ context.Context, id string, _, _, _, _ string, _ *pb.ModelOptions, _ bool) (*model.Model, error) {
+				return model.NewModelWithClient(id, "test://stale", rec), nil
+			})
+			cfg := identityModelCfg()
+			cfg.Backend = "llama-cpp"
+			cfg.Model = "native-test.gguf"
+			flags := config.FLAG_DECISIONS
+			cfg.KnownUsecases = &flags
+			app := config.NewApplicationConfig(config.WithSystemState(state))
+			marked := make(chan struct{}, 1)
+			gallery.SetStaleBackendNotifier(func() { marked <- struct{}{} })
+			DeferCleanup(func() { gallery.SetStaleBackendNotifier(nil) })
+			runner := backend.NewDecisionRunner(cfg.Name, func(string) *config.ModelConfig { return &cfg }, loader, app)
+			req := &schema.SystemOneRequest{State: json.RawMessage(`"x"`), Questions: map[string]schema.SystemOneQuestion{"q": {Type: "noul"}}}
+			_, err := runner.Decide(context.Background(), req)
+			Expect(err).To(HaveOccurred())
+			Expect(status.Code(err)).To(Equal(codes.Unimplemented))
+			Expect(err.Error()).To(ContainSubstring("older than this server"))
+			Expect(err.Error()).To(ContainSubstring("llama-cpp"))
+			Expect(err.Error()).NotTo(ContainSubstring("known_usecases"))
+			Eventually(marked).Should(Receive())
+		}
+	})
+
+	It("leaves other backend errors unchanged and does not mark the backend", func() {
+		rec := &nativeDecisionBackend{failWith: status.Error(codes.InvalidArgument, "bad question")}
+		state := &system.SystemState{}
+		loader := model.NewModelLoader(state)
+		loader.SetModelRouter(func(_ context.Context, id string, _, _, _, _ string, _ *pb.ModelOptions, _ bool) (*model.Model, error) {
+			return model.NewModelWithClient(id, "test://other", rec), nil
+		})
+		cfg := identityModelCfg()
+		cfg.Backend = "vllm-cpp"
+		cfg.Model = "native-test.gguf"
+		flags := config.FLAG_DECISIONS
+		cfg.KnownUsecases = &flags
+		app := config.NewApplicationConfig(config.WithSystemState(state))
+		notified := false
+		gallery.SetStaleBackendNotifier(func() { notified = true })
+		DeferCleanup(func() { gallery.SetStaleBackendNotifier(nil) })
+		runner := backend.NewDecisionRunner(cfg.Name, func(string) *config.ModelConfig { return &cfg }, loader, app)
+		req := &schema.SystemOneRequest{State: json.RawMessage(`"x"`), Questions: map[string]schema.SystemOneQuestion{"q": {Type: "noul"}}}
+		_, err := runner.Decide(context.Background(), req)
+		Expect(status.Code(err)).To(Equal(codes.InvalidArgument))
+		Expect(notified).To(BeFalse())
+	})
 })
@@ -0,0 +1,51 @@
+package backend
+
+import (
+	"strings"
+
+	"github.com/mudler/LocalAI/core/gallery"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// staleDecisionMessage is returned in place of the raw backend error when a
+// backend answers a decision request as a plain Score call. Constant text on
+// purpose: it never echoes backend or request content.
+const staleDecisionMessage = "the backend is older than this server and has no native decision support; " +
+	"upgrade the %q backend (see /backends/upgrades), then retry"
+
+// A decision request is a Score call with question_type "systemone". A backend
+// that knows the field never returns the two errors below for it: it either
+// runs the decision or answers UNIMPLEMENTED. A backend built before native
+// decisions ignores the unknown field and runs plain scoring, which fails with
+// one of these messages. So they identify an old backend, not a bad model
+// configuration, and adding "score" to known_usecases is not the fix.
+var staleDecisionSignatures = []string{
+	"Score was not enabled when the model was loaded",
+	"candidates must be non-empty",
+}
+
+// staleDecisionBackendError reports whether err, returned for a decision
+// request, comes from a backend that predates native decisions.
+func staleDecisionBackendError(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := status.Convert(err).Message()
+	for _, sig := range staleDecisionSignatures {
+		if strings.Contains(msg, sig) {
+			return true
+		}
+	}
+	return false
+}
+
+// handleStaleDecisionBackend turns the old-backend failure into a plain
+// error and flags the backend in the upgrade check. Other errors pass through.
+func handleStaleDecisionBackend(backendName string, err error) error {
+	if !staleDecisionBackendError(err) {
+		return err
+	}
+	gallery.MarkBackendStale(backendName, "backend does not support native decisions (older than this server)")
+	return status.Errorf(codes.Unimplemented, staleDecisionMessage, backendName)
+}
@@ -72,7 +72,7 @@ func ModelSystemOne(requestJSON string, loader *model.ModelLoader, modelConfig c
 			})
 		}
 		if err != nil {
-			return "", err
+			return "", handleStaleDecisionBackend(modelConfig.Backend, err)
 		}
 		return resp.GetResponseJson(), nil
 	}, nil
@@ -0,0 +1,83 @@
+package gallery
+
+import (
+	"sync"
+	"time"
+)
+
+// A backend can be current by tag, version and digest and still be too old
+// for the server. The gallery "latest" image of a backend can lag behind the
+// server code, so a tag or digest comparison reports "up to date" while the
+// backend lacks a feature the server relies on. The server cannot ask an
+// unloaded backend for its features, so it learns this when a request proves
+// it: the request carries a field only a new backend understands, and an old
+// backend answers as if the field were absent.
+//
+// MarkBackendStale records such an observation. CheckUpgradesAgainst then
+// reports every backend installed before that moment as upgradeable, with the
+// reason. Reinstalling the backend (which sets a newer InstalledAt) drops the
+// mark, so no explicit clearing is needed. If the reinstalled image is still
+// old, the next failing request marks it again.
+//
+// The marks are per process. In distributed mode each frontend learns from the
+// requests it serves.
+
+type staleMark struct {
+	at     time.Time
+	reason string
+}
+
+var (
+	staleMu       sync.Mutex
+	staleBackends = map[string]staleMark{}
+	staleNotify   func()
+)
+
+// MarkBackendStale records that the backend configured as name (a concrete
+// backend name or its alias, for example "llama-cpp") failed a request in a
+// way that shows it is older than the server. reason is shown to the operator.
+func MarkBackendStale(name, reason string) {
+	if name == "" {
+		return
+	}
+	staleMu.Lock()
+	staleBackends[name] = staleMark{at: time.Now(), reason: reason}
+	notify := staleNotify
+	staleMu.Unlock()
+	if notify != nil {
+		notify()
+	}
+}
+
+// SetStaleBackendNotifier registers a function called after each
+// MarkBackendStale, so the upgrade cache can refresh at once instead of at
+// the next periodic check. The function must not block.
+func SetStaleBackendNotifier(fn func()) {
+	staleMu.Lock()
+	staleNotify = fn
+	staleMu.Unlock()
+}
+
+// staleReason returns why the installed backend was marked stale, if a mark
+// matches its name or alias and was recorded after it was installed.
+func staleReason(md *BackendMetadata) (string, bool) {
+	if md == nil {
+		return "", false
+	}
+	installedAt, err := time.Parse(time.RFC3339, md.InstalledAt)
+	if err != nil {
+		// Unknown install time: treat the backend as installed before the mark.
+		installedAt = time.Time{}
+	}
+	staleMu.Lock()
+	defer staleMu.Unlock()
+	for _, key := range []string{md.Name, md.Alias} {
+		if key == "" {
+			continue
+		}
+		if m, ok := staleBackends[key]; ok && m.at.After(installedAt) {
+			return m.reason, true
+		}
+	}
+	return "", false
+}
@@ -23,6 +23,10 @@ type UpgradeInfo struct {
 	AvailableVersion string `json:"available_version"`
 	InstalledDigest  string `json:"installed_digest,omitempty"`
 	AvailableDigest  string `json:"available_digest,omitempty"`
+	// Reason says why the backend is flagged when version and digest alone
+	// would not flag it: the backend failed a request in a way that shows it
+	// is older than the server. Empty for ordinary version or digest upgrades.
+	Reason string `json:"reason,omitempty"`
 	// NodeDrift lists nodes whose installed version or digest differs from
 	// the cluster majority. Non-empty means the cluster has diverged and an
 	// upgrade will realign it. Empty in single-node mode.
@@ -129,6 +133,19 @@ func CheckUpgradesAgainst(ctx context.Context, galleries []config.Gallery, syste
 		installedDigest := installed.Metadata.Digest
 		galleryVersion := galleryEntry.Version

+		// A request showed this backend is older than the server, even though
+		// its tag, version or digest matches the gallery. See stale_backend.go.
+		if reason, stale := staleReason(installed.Metadata); stale {
+			result[installed.Metadata.Name] = UpgradeInfo{
+				BackendName:      installed.Metadata.Name,
+				InstalledVersion: installedVersion,
+				AvailableVersion: galleryVersion,
+				InstalledDigest:  installedDigest,
+				Reason:           reason,
+			}
+			continue
+		}
+
 		// Detect cluster drift: does every node report the same version+digest?
 		// In single-node mode this stays empty (Nodes is nil).
 		majority, drift := summarizeNodeDrift(installed.Nodes)
@@ -144,6 +144,71 @@ var _ = Describe("Upgrade Detection and Execution", func() {
 		})
 	})

+	Describe("stale backend marks", func() {
+		// Backend names are unique per test: the marks live in the process.
+		installAliased := func(name, alias, installedAt string) {
+			dir := filepath.Join(backendsPath, name)
+			Expect(os.MkdirAll(dir, 0750)).To(Succeed())
+			Expect(os.WriteFile(filepath.Join(dir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755)).To(Succeed())
+			data, err := json.Marshal(BackendMetadata{Name: name, Alias: alias, InstalledAt: installedAt})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed())
+			writeGalleryYAML([]GalleryBackend{{
+				Metadata: Metadata{Name: name},
+				URI:      filepath.Join(tempDir, "some-source"),
+			}})
+		}
+
+		It("flags a backend that is current by tag once a request proves it is old", func() {
+			installAliased("stale-cpp", "stale-alias", time.Now().Add(-time.Hour).Format(time.RFC3339))
+
+			upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(upgrades).To(BeEmpty())
+
+			MarkBackendStale("stale-alias", "no native decisions")
+			upgrades, err = CheckBackendUpgrades(context.Background(), galleries, systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(upgrades).To(HaveKey("stale-cpp"))
+			Expect(upgrades["stale-cpp"].Reason).To(Equal("no native decisions"))
+		})
+
+		It("matches a mark on the concrete backend name", func() {
+			installAliased("stale-named", "", time.Now().Add(-time.Hour).Format(time.RFC3339))
+			MarkBackendStale("stale-named", "old")
+			upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(upgrades).To(HaveKey("stale-named"))
+		})
+
+		It("drops the mark once the backend is reinstalled after it", func() {
+			installAliased("stale-fresh", "stale-fresh-alias", time.Now().Add(-time.Hour).Format(time.RFC3339))
+			MarkBackendStale("stale-fresh-alias", "old")
+			installAliased("stale-fresh", "stale-fresh-alias", time.Now().Add(time.Minute).Format(time.RFC3339))
+
+			upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(upgrades).To(BeEmpty())
+		})
+
+		It("ignores marks for other backends and empty names", func() {
+			installAliased("stale-other", "", time.Now().Add(-time.Hour).Format(time.RFC3339))
+			MarkBackendStale("", "ignored")
+			MarkBackendStale("not-installed", "ignored")
+			upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(upgrades).To(BeEmpty())
+		})
+
+		It("calls the notifier on each mark", func() {
+			calls := 0
+			SetStaleBackendNotifier(func() { calls++ })
+			DeferCleanup(func() { SetStaleBackendNotifier(nil) })
+			MarkBackendStale("stale-notify", "old")
+			Expect(calls).To(Equal(1))
+		})
+	})
+
 	// CheckUpgradesAgainst is the entry point used by DistributedBackendManager.
 	// It takes installed backends directly — typically aggregated from workers —
 	// instead of reading the frontend filesystem. These tests exercise drift
@@ -290,6 +290,15 @@ accept 2–10 levels; this backend-specific limit does not constrain vllm-cpp.
 Older forks without native decision support return 501. Missing decision metadata
 also returns 501, while backend invalid requests return 400.

+A `llama-cpp` backend image built before native decisions cannot answer these
+requests, even when the gallery reports it as up to date. LocalAI detects this
+from the failed request: it returns 501 with a message that the backend is older
+than the server, and `GET /backends/upgrades` lists the backend with a `reason`
+field. Upgrade the backend, or install a current build such as the
+`-development` variant. Adding `score` to `known_usecases` does not help. The
+flag clears when the backend is reinstalled; if the reinstalled image is still
+old, the next decision request flags it again.
+
 ### Bounded image input

 The public routes and internal decision validator share an image contract. Supply