mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
2b61e4bc1d
CheckUpgradesAgainst resolved gallery entries through AvailableBackends, which drops every entry the *local* host cannot run. In distributed mode the host running the check is a CPU-only controller while the GPU backends live on worker nodes, so FindGalleryElement returned nil for every cuda/rocm/l4t entry and those backends were silently skipped. Measured on a live cluster: GET /backends reported 48 installed backends, POST /backends/upgrades/check evaluated 5 — all of them plain or cpu-prefixed. The 43 skipped were all hardware-specific builds. As a result cuda13-nvidia-l4t-arm64-longcat-video-development stayed at sha256:0b8dc851 while the registry tag held sha256:38dae6ff, and a cuDNN packaging fix sat unnoticed on a GPU worker for two days. Every name looked up here is already installed somewhere in the cluster, so hardware compatibility was decided at install time; re-deciding it against the controller is wrong. Switch both CheckUpgradesAgainst and UpgradeBackend to AvailableBackendsUnfiltered. Assisted-by: Claude Code:claude-opus-4-8[1m] [Read] [Edit] [Bash] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
514 lines
17 KiB
Go
514 lines
17 KiB
Go
package gallery_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/mudler/LocalAI/core/config"
|
|
. "github.com/mudler/LocalAI/core/gallery"
|
|
"github.com/mudler/LocalAI/pkg/model"
|
|
"github.com/mudler/LocalAI/pkg/system"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
var _ = Describe("Upgrade Detection and Execution", func() {
|
|
var (
|
|
tempDir string
|
|
backendsPath string
|
|
galleryPath string
|
|
systemState *system.SystemState
|
|
galleries []config.Gallery
|
|
)
|
|
|
|
// installBackendWithVersion creates a fake installed backend directory with
|
|
// the given name, version, and optional run.sh content.
|
|
installBackendWithVersion := func(name, version string, runContent ...string) {
|
|
dir := filepath.Join(backendsPath, name)
|
|
Expect(os.MkdirAll(dir, 0750)).To(Succeed())
|
|
|
|
content := "#!/bin/sh\necho ok"
|
|
if len(runContent) > 0 {
|
|
content = runContent[0]
|
|
}
|
|
Expect(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(content), 0755)).To(Succeed())
|
|
|
|
metadata := BackendMetadata{
|
|
Name: name,
|
|
Version: version,
|
|
InstalledAt: time.Now().Format(time.RFC3339),
|
|
}
|
|
data, err := json.MarshalIndent(metadata, "", " ")
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed())
|
|
}
|
|
|
|
// writeGalleryYAML writes a gallery YAML file with the given backends.
|
|
writeGalleryYAML := func(backends []GalleryBackend) {
|
|
data, err := yaml.Marshal(backends)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(os.WriteFile(galleryPath, data, 0644)).To(Succeed())
|
|
}
|
|
|
|
BeforeEach(func() {
|
|
var err error
|
|
tempDir, err = os.MkdirTemp("", "upgrade-test-*")
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
backendsPath = tempDir
|
|
|
|
galleryPath = filepath.Join(tempDir, "gallery.yaml")
|
|
|
|
// Write a default empty gallery
|
|
writeGalleryYAML([]GalleryBackend{})
|
|
|
|
galleries = []config.Gallery{
|
|
{
|
|
Name: "test-gallery",
|
|
URL: "file://" + galleryPath,
|
|
},
|
|
}
|
|
|
|
systemState, err = system.GetSystemState(
|
|
system.WithBackendPath(backendsPath),
|
|
)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
})
|
|
|
|
AfterEach(func() {
|
|
os.RemoveAll(tempDir)
|
|
})
|
|
|
|
Describe("CheckBackendUpgrades", func() {
|
|
It("should detect upgrade when gallery version differs from installed version", func() {
|
|
// Install a backend at v1.0.0
|
|
installBackendWithVersion("my-backend", "1.0.0")
|
|
|
|
// Gallery advertises v2.0.0
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{
|
|
Name: "my-backend",
|
|
},
|
|
URI: filepath.Join(tempDir, "some-source"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(HaveKey("my-backend"))
|
|
Expect(upgrades["my-backend"].InstalledVersion).To(Equal("1.0.0"))
|
|
Expect(upgrades["my-backend"].AvailableVersion).To(Equal("2.0.0"))
|
|
})
|
|
|
|
It("should NOT flag upgrade when versions match", func() {
|
|
installBackendWithVersion("my-backend", "2.0.0")
|
|
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{
|
|
Name: "my-backend",
|
|
},
|
|
URI: filepath.Join(tempDir, "some-source"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(BeEmpty())
|
|
})
|
|
|
|
It("should skip backends without version info and without OCI digest", func() {
|
|
// Install without version
|
|
installBackendWithVersion("my-backend", "")
|
|
|
|
// Gallery also without version
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{
|
|
Name: "my-backend",
|
|
},
|
|
URI: filepath.Join(tempDir, "some-source"),
|
|
},
|
|
})
|
|
|
|
upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(BeEmpty())
|
|
})
|
|
})
|
|
|
|
// CheckUpgradesAgainst is the entry point used by DistributedBackendManager.
|
|
// It takes installed backends directly — typically aggregated from workers —
|
|
// instead of reading the frontend filesystem. These tests exercise drift
|
|
// detection, which is the feature the distributed path relies on.
|
|
Describe("CheckUpgradesAgainst (distributed)", func() {
|
|
It("flags upgrade when cluster nodes disagree on version, even if gallery matches majority", func() {
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{Name: "my-backend"},
|
|
URI: filepath.Join(tempDir, "some-source"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
installed := SystemBackends{
|
|
"my-backend": SystemBackend{
|
|
Name: "my-backend",
|
|
Metadata: &BackendMetadata{Name: "my-backend", Version: "2.0.0"},
|
|
Nodes: []NodeBackendRef{
|
|
{NodeID: "a", NodeName: "worker-1", Version: "2.0.0"},
|
|
{NodeID: "b", NodeName: "worker-2", Version: "2.0.0"},
|
|
{NodeID: "c", NodeName: "worker-3", Version: "1.0.0"}, // drift
|
|
},
|
|
},
|
|
}
|
|
|
|
upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, systemState, installed)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(HaveKey("my-backend"))
|
|
info := upgrades["my-backend"]
|
|
Expect(info.AvailableVersion).To(Equal("2.0.0"))
|
|
Expect(info.NodeDrift).To(HaveLen(1))
|
|
Expect(info.NodeDrift[0].NodeName).To(Equal("worker-3"))
|
|
Expect(info.NodeDrift[0].Version).To(Equal("1.0.0"))
|
|
})
|
|
|
|
It("does not flag upgrade when all nodes agree and match gallery", func() {
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{Name: "my-backend"},
|
|
URI: filepath.Join(tempDir, "some-source"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
installed := SystemBackends{
|
|
"my-backend": SystemBackend{
|
|
Name: "my-backend",
|
|
Metadata: &BackendMetadata{Name: "my-backend", Version: "2.0.0"},
|
|
Nodes: []NodeBackendRef{
|
|
{NodeID: "a", NodeName: "worker-1", Version: "2.0.0"},
|
|
{NodeID: "b", NodeName: "worker-2", Version: "2.0.0"},
|
|
},
|
|
},
|
|
}
|
|
|
|
upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, systemState, installed)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(BeEmpty())
|
|
})
|
|
|
|
It("surfaces empty-installed-version path the old distributed code silently missed", func() {
|
|
// Simulates the real-world bug: worker has a backend, its version
|
|
// is empty (pre-tracking or OCI-pinned-to-latest), gallery has a
|
|
// version. Pre-fix CheckUpgrades returned nothing; now it surfaces.
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{Name: "my-backend"},
|
|
URI: filepath.Join(tempDir, "some-source"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
installed := SystemBackends{
|
|
"my-backend": SystemBackend{
|
|
Name: "my-backend",
|
|
Metadata: &BackendMetadata{Name: "my-backend"},
|
|
Nodes: []NodeBackendRef{
|
|
{NodeID: "a", NodeName: "worker-1"},
|
|
},
|
|
},
|
|
}
|
|
|
|
upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, systemState, installed)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(HaveKey("my-backend"))
|
|
Expect(upgrades["my-backend"].InstalledVersion).To(BeEmpty())
|
|
Expect(upgrades["my-backend"].AvailableVersion).To(Equal("2.0.0"))
|
|
})
|
|
|
|
// Dev-aware suppression: when `<X>-development` is installed it
|
|
// stands in for the stable `<X>` via alias resolution. Auto-upgrade
|
|
// must never reintroduce the stable variant alongside the dev one,
|
|
// because the install would land on disk and (depending on
|
|
// preference tokens) either shadow the dev pick or sit unused next
|
|
// to it. These tests fix CheckUpgradesAgainst to honor that.
|
|
// Names are kept generic ("my-backend") so the capability filter
|
|
// in AvailableBackends doesn't drop them on a CPU-only test host.
|
|
It("suppresses non-dev candidate when its -development counterpart is installed", func() {
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{Name: "my-backend"},
|
|
URI: filepath.Join(tempDir, "stable"),
|
|
Version: "2.0.0",
|
|
},
|
|
{
|
|
Metadata: Metadata{Name: "my-backend-development"},
|
|
URI: filepath.Join(tempDir, "dev"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
installed := SystemBackends{
|
|
"my-backend-development": SystemBackend{
|
|
Name: "my-backend-development",
|
|
Metadata: &BackendMetadata{
|
|
Name: "my-backend-development",
|
|
Version: "1.0.0",
|
|
},
|
|
},
|
|
}
|
|
|
|
upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, systemState, installed)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(HaveKey("my-backend-development"))
|
|
Expect(upgrades).NotTo(HaveKey("my-backend"))
|
|
})
|
|
|
|
It("dev variant wins even when non-dev is also present (vestigial state)", func() {
|
|
// Either via legacy state, manual install, or a worker still
|
|
// emitting synthetic aliases, the non-dev row may be present
|
|
// alongside the dev one. Auto-upgrade must still keep its
|
|
// hands off the non-dev — installing the stable variant on
|
|
// top of the user's explicit dev pick is exactly what the
|
|
// alias drop-in promise forbids. Users who genuinely want
|
|
// the non-dev upgraded can trigger it manually via
|
|
// /api/backends/upgrade/<name>.
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{Name: "my-backend"},
|
|
URI: filepath.Join(tempDir, "stable"),
|
|
Version: "2.0.0",
|
|
},
|
|
{
|
|
Metadata: Metadata{Name: "my-backend-development"},
|
|
URI: filepath.Join(tempDir, "dev"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
installed := SystemBackends{
|
|
"my-backend": SystemBackend{
|
|
Name: "my-backend",
|
|
Metadata: &BackendMetadata{
|
|
Name: "my-backend",
|
|
Version: "1.0.0",
|
|
},
|
|
},
|
|
"my-backend-development": SystemBackend{
|
|
Name: "my-backend-development",
|
|
Metadata: &BackendMetadata{
|
|
Name: "my-backend-development",
|
|
Version: "1.0.0",
|
|
},
|
|
},
|
|
}
|
|
|
|
upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, systemState, installed)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(HaveKey("my-backend-development"))
|
|
Expect(upgrades).NotTo(HaveKey("my-backend"))
|
|
})
|
|
|
|
It("ignores synthetic alias rows whose key differs from Metadata.Name", func() {
|
|
// ListSystemBackends emits an extra row keyed by the alias name
|
|
// that re-uses the chosen concrete's metadata pointer. Pre-fix
|
|
// this row caused a duplicate gallery lookup in single-node
|
|
// (harmless by accident) and a phantom upgrade in distributed
|
|
// mode (real bug — the wire-reconstructed row carries
|
|
// Metadata.Name = alias and resolves against an unrelated entry).
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{Name: "my-alias"},
|
|
URI: filepath.Join(tempDir, "stable-meta"),
|
|
Version: "2.0.0",
|
|
},
|
|
{
|
|
Metadata: Metadata{Name: "my-backend-development"},
|
|
URI: filepath.Join(tempDir, "dev"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
devMeta := &BackendMetadata{
|
|
Name: "my-backend-development",
|
|
Version: "1.0.0",
|
|
Alias: "my-alias",
|
|
}
|
|
installed := SystemBackends{
|
|
"my-backend-development": SystemBackend{
|
|
Name: "my-backend-development",
|
|
Metadata: devMeta,
|
|
},
|
|
// Synthetic alias row: key != Metadata.Name.
|
|
"my-alias": SystemBackend{
|
|
Name: "my-alias",
|
|
Metadata: devMeta,
|
|
},
|
|
}
|
|
|
|
upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, systemState, installed)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(HaveKey("my-backend-development"))
|
|
Expect(upgrades).NotTo(HaveKey("my-alias"))
|
|
})
|
|
|
|
// Hardware-specific backends live on GPU worker nodes while the
|
|
// controller is typically a CPU-only pod. The gallery candidate set
|
|
// must therefore NOT be filtered by the controller's own capability:
|
|
// doing so drops every cuda/rocm/l4t entry, FindGalleryElement
|
|
// returns nil, and the backend is silently skipped. Observed live:
|
|
// 48 installed backends, only 5 (all CPU) ever evaluated, while
|
|
// cuda13-nvidia-l4t-arm64-longcat-video-development sat two versions
|
|
// behind on a worker.
|
|
It("flags a GPU-only backend installed on a worker even though the controller is CPU-only", func() {
|
|
cpuOnlyState := system.NewCapabilityState("default",
|
|
system.WithBackendPath(backendsPath))
|
|
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{Name: "cuda13-nvidia-l4t-arm64-longcat-video-development"},
|
|
URI: filepath.Join(tempDir, "gpu-source"),
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
installed := SystemBackends{
|
|
"cuda13-nvidia-l4t-arm64-longcat-video-development": SystemBackend{
|
|
Name: "cuda13-nvidia-l4t-arm64-longcat-video-development",
|
|
Metadata: &BackendMetadata{
|
|
Name: "cuda13-nvidia-l4t-arm64-longcat-video-development",
|
|
Version: "1.0.0",
|
|
},
|
|
Nodes: []NodeBackendRef{
|
|
{NodeID: "a", NodeName: "gpu-worker-1", Version: "1.0.0"},
|
|
},
|
|
},
|
|
}
|
|
|
|
upgrades, err := CheckUpgradesAgainst(context.Background(), galleries, cpuOnlyState, installed)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(upgrades).To(HaveKey("cuda13-nvidia-l4t-arm64-longcat-video-development"))
|
|
info := upgrades["cuda13-nvidia-l4t-arm64-longcat-video-development"]
|
|
Expect(info.InstalledVersion).To(Equal("1.0.0"))
|
|
Expect(info.AvailableVersion).To(Equal("2.0.0"))
|
|
})
|
|
})
|
|
|
|
Describe("UpgradeBackend", func() {
|
|
It("should replace backend directory and update metadata", func() {
|
|
// Install v1
|
|
installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1")
|
|
|
|
// Create a source directory with v2 content
|
|
srcDir := filepath.Join(tempDir, "v2-source")
|
|
Expect(os.MkdirAll(srcDir, 0750)).To(Succeed())
|
|
Expect(os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho v2"), 0755)).To(Succeed())
|
|
|
|
// Gallery points to the v2 source dir
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{
|
|
Name: "my-backend",
|
|
},
|
|
URI: srcDir,
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
ml := model.NewModelLoader(systemState)
|
|
err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil, false)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// Verify run.sh was updated
|
|
content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh"))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(string(content)).To(Equal("#!/bin/sh\necho v2"))
|
|
|
|
// Verify metadata was updated
|
|
metaData, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "metadata.json"))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
var meta BackendMetadata
|
|
Expect(json.Unmarshal(metaData, &meta)).To(Succeed())
|
|
Expect(meta.Version).To(Equal("2.0.0"))
|
|
Expect(meta.Name).To(Equal("my-backend"))
|
|
})
|
|
|
|
It("should restore backup on failure", func() {
|
|
// Install v1
|
|
installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1")
|
|
|
|
// Gallery points to a nonexistent path (no run.sh will be found)
|
|
nonExistentDir := filepath.Join(tempDir, "does-not-exist")
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{
|
|
Name: "my-backend",
|
|
},
|
|
URI: nonExistentDir,
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
ml := model.NewModelLoader(systemState)
|
|
err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil, false)
|
|
Expect(err).To(HaveOccurred())
|
|
|
|
// Verify v1 is still intact
|
|
content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh"))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(string(content)).To(Equal("#!/bin/sh\necho v1"))
|
|
|
|
// Verify metadata still says v1
|
|
metaData, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "metadata.json"))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
var meta BackendMetadata
|
|
Expect(json.Unmarshal(metaData, &meta)).To(Succeed())
|
|
Expect(meta.Version).To(Equal("1.0.0"))
|
|
})
|
|
|
|
// Regression: an earlier version of UpgradeBackend wrote the
|
|
// downloaded bytes to disk without going through
|
|
// backendDownloadOptions, so the gallery's verification policy
|
|
// (and strict-integrity gate) didn't apply on upgrade. This test
|
|
// pins the upgrade path to the same integrity gate as installs:
|
|
// strict mode + an OCI URI without a verification: block must
|
|
// hard-fail *before* anything is downloaded or swapped in.
|
|
It("should refuse to upgrade an OCI backend that bypasses integrity in strict mode", func() {
|
|
installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1")
|
|
|
|
// OCI URI, no Gallery.Verification → backendDownloadOptions
|
|
// returns a strict-integrity error before any network call.
|
|
writeGalleryYAML([]GalleryBackend{
|
|
{
|
|
Metadata: Metadata{
|
|
Name: "my-backend",
|
|
},
|
|
URI: "oci://example.invalid/missing:never-fetched",
|
|
Version: "2.0.0",
|
|
},
|
|
})
|
|
|
|
ml := model.NewModelLoader(systemState)
|
|
err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil, true)
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring("strict integrity"))
|
|
|
|
// The installed v1 must be untouched — the upgrade should
|
|
// have aborted before writing anything.
|
|
content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh"))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(string(content)).To(Equal("#!/bin/sh\necho v1"))
|
|
Expect(filepath.Join(backendsPath, "my-backend.upgrade-tmp")).NotTo(BeAnExistingFile())
|
|
Expect(filepath.Join(backendsPath, "my-backend.backup")).NotTo(BeAnExistingFile())
|
|
})
|
|
})
|
|
})
|