diff --git a/core/gallery/upgrade.go b/core/gallery/upgrade.go index 43d9d9d30..7abc65a43 100644 --- a/core/gallery/upgrade.go +++ b/core/gallery/upgrade.go @@ -58,7 +58,15 @@ func CheckBackendUpgrades(ctx context.Context, galleries []config.Gallery, syste // row is flagged upgradeable regardless of whether any node matches the gallery // — next Upgrade All realigns the cluster. NodeDrift lists the outliers. func CheckUpgradesAgainst(ctx context.Context, galleries []config.Gallery, systemState *system.SystemState, installedBackends SystemBackends) (map[string]UpgradeInfo, error) { - galleryBackends, err := AvailableBackends(galleries, systemState) + // Unfiltered on purpose. AvailableBackends drops every gallery entry the + // *local* host cannot run, and in distributed mode the host running this + // check is a CPU-only controller while the GPU backends live on worker + // nodes. Filtering there made FindGalleryElement return nil for every + // cuda/rocm/l4t entry, so those backends were silently skipped and never + // reported an upgrade. Every name we look up here is already installed + // somewhere in the cluster, so hardware compatibility has been decided at + // install time and re-deciding it against the controller is wrong. + galleryBackends, err := AvailableBackendsUnfiltered(galleries, systemState) if err != nil { return nil, fmt.Errorf("failed to list available backends: %w", err) } @@ -254,8 +262,10 @@ func UpgradeBackend(ctx context.Context, systemState *system.SystemState, modelL return UpgradeBackend(ctx, systemState, modelLoader, galleries, installed.Metadata.MetaBackendFor, downloadStatus, requireIntegrity) } - // Find the gallery entry - galleryBackends, err := AvailableBackends(galleries, systemState) + // Find the gallery entry. Unfiltered for the same reason as the check + // above: backendName is already installed here, so the capability filter + // can only reject an entry we must be able to resolve. + galleryBackends, err := AvailableBackendsUnfiltered(galleries, systemState) if err != nil { return fmt.Errorf("failed to list available backends: %w", err) } diff --git a/core/gallery/upgrade_test.go b/core/gallery/upgrade_test.go index 25d6802c4..ae1be8fcc 100644 --- a/core/gallery/upgrade_test.go +++ b/core/gallery/upgrade_test.go @@ -359,6 +359,47 @@ var _ = Describe("Upgrade Detection and Execution", func() { Expect(upgrades).To(HaveKey("my-backend-development")) Expect(upgrades).NotTo(HaveKey("my-alias")) }) + + // Hardware-specific backends live on GPU worker nodes while the + // controller is typically a CPU-only pod. The gallery candidate set + // must therefore NOT be filtered by the controller's own capability: + // doing so drops every cuda/rocm/l4t entry, FindGalleryElement + // returns nil, and the backend is silently skipped. Observed live: + // 48 installed backends, only 5 (all CPU) ever evaluated, while + // cuda13-nvidia-l4t-arm64-longcat-video-development sat two versions + // behind on a worker. + It("flags a GPU-only backend installed on a worker even though the controller is CPU-only", func() { + cpuOnlyState := system.NewCapabilityState("default", + system.WithBackendPath(backendsPath)) + + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{Name: "cuda13-nvidia-l4t-arm64-longcat-video-development"}, + URI: filepath.Join(tempDir, "gpu-source"), + Version: "2.0.0", + }, + }) + + installed := SystemBackends{ + "cuda13-nvidia-l4t-arm64-longcat-video-development": SystemBackend{ + Name: "cuda13-nvidia-l4t-arm64-longcat-video-development", + Metadata: &BackendMetadata{ + Name: "cuda13-nvidia-l4t-arm64-longcat-video-development", + Version: "1.0.0", + }, + Nodes: []NodeBackendRef{ + {NodeID: "a", NodeName: "gpu-worker-1", Version: "1.0.0"}, + }, + }, + } + + upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, cpuOnlyState, installed) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(HaveKey("cuda13-nvidia-l4t-arm64-longcat-video-development")) + info := upgrades["cuda13-nvidia-l4t-arm64-longcat-video-development"] + Expect(info.InstalledVersion).To(Equal("1.0.0")) + Expect(info.AvailableVersion).To(Equal("2.0.0")) + }) }) Describe("UpgradeBackend", func() {