From 2b61e4bc1d84f4fad786ae024f74e12f059b7389 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:28:34 +0200 Subject: [PATCH] fix(upgrade-check): don't filter upgrade candidates by controller capability (#11024) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CheckUpgradesAgainst resolved gallery entries through AvailableBackends, which drops every entry the *local* host cannot run. In distributed mode the host running the check is a CPU-only controller while the GPU backends live on worker nodes, so FindGalleryElement returned nil for every cuda/rocm/l4t entry and those backends were silently skipped. Measured on a live cluster: GET /backends reported 48 installed backends, POST /backends/upgrades/check evaluated 5 — all of them plain or cpu-prefixed. The 43 skipped were all hardware-specific builds. As a result cuda13-nvidia-l4t-arm64-longcat-video-development stayed at sha256:0b8dc851 while the registry tag held sha256:38dae6ff, and a cuDNN packaging fix sat unnoticed on a GPU worker for two days. Every name looked up here is already installed somewhere in the cluster, so hardware compatibility was decided at install time; re-deciding it against the controller is wrong. Switch both CheckUpgradesAgainst and UpgradeBackend to AvailableBackendsUnfiltered. Assisted-by: Claude Code:claude-opus-4-8[1m] [Read] [Edit] [Bash] Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/gallery/upgrade.go | 16 +++++++++++--- core/gallery/upgrade_test.go | 41 ++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/core/gallery/upgrade.go b/core/gallery/upgrade.go index 43d9d9d30..7abc65a43 100644 --- a/core/gallery/upgrade.go +++ b/core/gallery/upgrade.go @@ -58,7 +58,15 @@ func CheckBackendUpgrades(ctx context.Context, galleries []config.Gallery, syste // row is flagged upgradeable regardless of whether any node matches the gallery // — next Upgrade All realigns the cluster. NodeDrift lists the outliers. func CheckUpgradesAgainst(ctx context.Context, galleries []config.Gallery, systemState *system.SystemState, installedBackends SystemBackends) (map[string]UpgradeInfo, error) { - galleryBackends, err := AvailableBackends(galleries, systemState) + // Unfiltered on purpose. AvailableBackends drops every gallery entry the + // *local* host cannot run, and in distributed mode the host running this + // check is a CPU-only controller while the GPU backends live on worker + // nodes. Filtering there made FindGalleryElement return nil for every + // cuda/rocm/l4t entry, so those backends were silently skipped and never + // reported an upgrade. Every name we look up here is already installed + // somewhere in the cluster, so hardware compatibility has been decided at + // install time and re-deciding it against the controller is wrong. + galleryBackends, err := AvailableBackendsUnfiltered(galleries, systemState) if err != nil { return nil, fmt.Errorf("failed to list available backends: %w", err) } @@ -254,8 +262,10 @@ func UpgradeBackend(ctx context.Context, systemState *system.SystemState, modelL return UpgradeBackend(ctx, systemState, modelLoader, galleries, installed.Metadata.MetaBackendFor, downloadStatus, requireIntegrity) } - // Find the gallery entry - galleryBackends, err := AvailableBackends(galleries, systemState) + // Find the gallery entry. Unfiltered for the same reason as the check + // above: backendName is already installed here, so the capability filter + // can only reject an entry we must be able to resolve. + galleryBackends, err := AvailableBackendsUnfiltered(galleries, systemState) if err != nil { return fmt.Errorf("failed to list available backends: %w", err) } diff --git a/core/gallery/upgrade_test.go b/core/gallery/upgrade_test.go index 25d6802c4..ae1be8fcc 100644 --- a/core/gallery/upgrade_test.go +++ b/core/gallery/upgrade_test.go @@ -359,6 +359,47 @@ var _ = Describe("Upgrade Detection and Execution", func() { Expect(upgrades).To(HaveKey("my-backend-development")) Expect(upgrades).NotTo(HaveKey("my-alias")) }) + + // Hardware-specific backends live on GPU worker nodes while the + // controller is typically a CPU-only pod. The gallery candidate set + // must therefore NOT be filtered by the controller's own capability: + // doing so drops every cuda/rocm/l4t entry, FindGalleryElement + // returns nil, and the backend is silently skipped. Observed live: + // 48 installed backends, only 5 (all CPU) ever evaluated, while + // cuda13-nvidia-l4t-arm64-longcat-video-development sat two versions + // behind on a worker. + It("flags a GPU-only backend installed on a worker even though the controller is CPU-only", func() { + cpuOnlyState := system.NewCapabilityState("default", + system.WithBackendPath(backendsPath)) + + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{Name: "cuda13-nvidia-l4t-arm64-longcat-video-development"}, + URI: filepath.Join(tempDir, "gpu-source"), + Version: "2.0.0", + }, + }) + + installed := SystemBackends{ + "cuda13-nvidia-l4t-arm64-longcat-video-development": SystemBackend{ + Name: "cuda13-nvidia-l4t-arm64-longcat-video-development", + Metadata: &BackendMetadata{ + Name: "cuda13-nvidia-l4t-arm64-longcat-video-development", + Version: "1.0.0", + }, + Nodes: []NodeBackendRef{ + {NodeID: "a", NodeName: "gpu-worker-1", Version: "1.0.0"}, + }, + }, + } + + upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, cpuOnlyState, installed) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(HaveKey("cuda13-nvidia-l4t-arm64-longcat-video-development")) + info := upgrades["cuda13-nvidia-l4t-arm64-longcat-video-development"] + Expect(info.InstalledVersion).To(Equal("1.0.0")) + Expect(info.AvailableVersion).To(Equal("2.0.0")) + }) }) Describe("UpgradeBackend", func() {