From 7fd90419f5614b8602d20e5c0dd0307178e07523 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 03:21:00 +0000 Subject: [PATCH] fix(gallery): budget variant memory from RAM when a GPU reports no VRAM Variant selection read its memory budget from VRAM whenever a GPU capability was detected, and from system RAM only when none was. Apple Silicon satisfies the first branch and fails the premise: arm64 macs report the metal capability unconditionally, without probing anything, while TotalAvailableVRAM has no discrete VRAM pool to find and returns zero. The budget therefore came out as zero on every Mac. Zero drops every variant carrying a known size, so the base build was installed on all of them however much memory the machine had. The feature was inert on the platform, and silently: falling back to the base is a legitimate outcome, so nothing looked wrong. Take VRAM only when it is actually a number, and fall back to RAM otherwise. On a unified-memory host RAM is not an approximation of the budget, it is the budget, since the GPU shares it. A discrete GPU whose VRAM could not be read also lands on RAM, which overstates what the card holds but understates nothing the host has; the previous zero understated both. An unreadable RAM figure still yields zero and still installs the base, so a genuinely unknown host is not talked into a larger download. This is what turned tests-apple red: "installs a fitting variant's payload under the entry's own name" asserts on selection, and the runner resolved to the base because its budget was zero. The added specs pin the branch directly rather than relying on a macOS runner to notice again. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .../gallery/available_memory_internal_test.go | 73 +++++++++++++++++++ core/gallery/models.go | 11 ++- docs/content/features/model-gallery.md | 6 +- 3 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 core/gallery/available_memory_internal_test.go diff --git a/core/gallery/available_memory_internal_test.go b/core/gallery/available_memory_internal_test.go new file mode 100644 index 000000000..e62b13800 --- /dev/null +++ b/core/gallery/available_memory_internal_test.go @@ -0,0 +1,73 @@ +package gallery + +import ( + "os" + + "github.com/mudler/LocalAI/pkg/system" + "github.com/mudler/LocalAI/pkg/xsysinfo" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("availableModelMemory", func() { + // hostRAM is the figure the fallback resolves to. A host that cannot report + // its own RAM cannot exercise the fallback at all, so those specs skip + // rather than assert on a number that is legitimately unavailable. + hostRAM := func() uint64 { + ram, err := xsysinfo.GetSystemRAMInfo() + if err != nil || ram == nil || ram.Total == 0 { + Skip("this host does not report system RAM; the RAM fallback cannot be exercised here") + } + return ram.Total + } + + // stateWithCapability forces DetectedCapability for one spec. The state is + // built fresh so nothing is served from the capability cache. + stateWithCapability := func(capability string, vram uint64) *system.SystemState { + previous, had := os.LookupEnv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", capability)).To(Succeed()) + DeferCleanup(func() { + if had { + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", previous)).To(Succeed()) + return + } + Expect(os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY")).To(Succeed()) + }) + + s := &system.SystemState{} + s.VRAM = vram + Expect(s.DetectedCapability()).To(Equal(capability), "the capability override did not take") + return s + } + + // A unified-memory host reports a GPU capability but no discrete VRAM pool. + // Apple Silicon is the case that matters: metal is reported unconditionally + // on arm64 macs while TotalAvailableVRAM finds nothing to add up. Reading + // that zero as the budget drops every sized variant and strands the host on + // the base build however much memory it has, which is what turned the macOS + // runner red. + It("falls back to system RAM when a detected GPU reports no VRAM", func() { + ram := hostRAM() + + got := availableModelMemory(stateWithCapability("metal", 0)) + + Expect(got).ToNot(BeZero(), "a unified-memory host was given a zero budget; every sized variant would be dropped") + Expect(got).To(Equal(ram)) + }) + + // A discrete GPU reports real VRAM, and that stays the budget: the model + // lives on the card, so system RAM is not what bounds it. + It("prefers VRAM when the GPU reports it", func() { + const vram = uint64(24) << 30 + + Expect(availableModelMemory(stateWithCapability("nvidia-cuda-12", vram))).To(Equal(vram)) + }) + + // With no GPU at all the model lives in system RAM. + It("uses system RAM when no GPU is detected", func() { + ram := hostRAM() + + Expect(availableModelMemory(stateWithCapability("default", 0))).To(Equal(ram)) + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index 18a314dfc..726fe07b8 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -287,16 +287,23 @@ const noGPUDetected = "default" // availableModelMemory reports how much memory a model may occupy on this host. // -// With a usable GPU the model lives in VRAM. Without one it lives in system +// With a discrete GPU the model lives in VRAM. Otherwise it lives in system // RAM, read through xsysinfo because that path is cgroup-aware: under // Kubernetes the container's limit, not the node's physical RAM, is what the // model actually gets. // +// A detected GPU whose VRAM reads as zero falls back to RAM rather than to +// zero. That is the normal shape of a unified-memory host, not an error: +// arm64 macs report the metal capability unconditionally, yet have no discrete +// VRAM pool for TotalAvailableVRAM to find, so the model's real budget is +// system RAM. Returning the zero here would strand every Mac on the base +// build no matter how much memory it has. +// // An unreadable RAM figure yields 0, which drops every variant with a known // requirement and installs the base. That is the safe direction: an unknown // host should not be talked into a larger download. func availableModelMemory(systemState *system.SystemState) uint64 { - if systemState.DetectedCapability() != noGPUDetected { + if systemState.DetectedCapability() != noGPUDetected && systemState.VRAM > 0 { return systemState.VRAM } ram, err := xsysinfo.GetSystemRAMInfo() diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 1c66967f2..b02ae336d 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -158,8 +158,10 @@ quantizations, or the same weights served by a different engine. Such an entry carries a `variants` list, and installing it normally lets LocalAI choose: - variants whose backend cannot run on this machine are dropped; -- variants that do not fit the available memory (VRAM on a GPU host, otherwise - system RAM) are dropped; +- variants that do not fit the available memory are dropped. That budget is + VRAM on a discrete-GPU host, and system RAM otherwise — including on + unified-memory machines such as Apple Silicon, where the GPU shares system + RAM and reports no separate VRAM pool; - the entry's own build is never dropped. It competes with whatever survived rather than waiting for everything else to fail, so an entry that is itself the largest build that fits keeps its own payload;