diff --git a/core/gallery/available_memory_internal_test.go b/core/gallery/available_memory_internal_test.go new file mode 100644 index 000000000..e62b13800 --- /dev/null +++ b/core/gallery/available_memory_internal_test.go @@ -0,0 +1,73 @@ +package gallery + +import ( + "os" + + "github.com/mudler/LocalAI/pkg/system" + "github.com/mudler/LocalAI/pkg/xsysinfo" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("availableModelMemory", func() { + // hostRAM is the figure the fallback resolves to. A host that cannot report + // its own RAM cannot exercise the fallback at all, so those specs skip + // rather than assert on a number that is legitimately unavailable. + hostRAM := func() uint64 { + ram, err := xsysinfo.GetSystemRAMInfo() + if err != nil || ram == nil || ram.Total == 0 { + Skip("this host does not report system RAM; the RAM fallback cannot be exercised here") + } + return ram.Total + } + + // stateWithCapability forces DetectedCapability for one spec. The state is + // built fresh so nothing is served from the capability cache. + stateWithCapability := func(capability string, vram uint64) *system.SystemState { + previous, had := os.LookupEnv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", capability)).To(Succeed()) + DeferCleanup(func() { + if had { + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", previous)).To(Succeed()) + return + } + Expect(os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY")).To(Succeed()) + }) + + s := &system.SystemState{} + s.VRAM = vram + Expect(s.DetectedCapability()).To(Equal(capability), "the capability override did not take") + return s + } + + // A unified-memory host reports a GPU capability but no discrete VRAM pool. + // Apple Silicon is the case that matters: metal is reported unconditionally + // on arm64 macs while TotalAvailableVRAM finds nothing to add up. Reading + // that zero as the budget drops every sized variant and strands the host on + // the base build however much memory it has, which is what turned the macOS + // runner red. + It("falls back to system RAM when a detected GPU reports no VRAM", func() { + ram := hostRAM() + + got := availableModelMemory(stateWithCapability("metal", 0)) + + Expect(got).ToNot(BeZero(), "a unified-memory host was given a zero budget; every sized variant would be dropped") + Expect(got).To(Equal(ram)) + }) + + // A discrete GPU reports real VRAM, and that stays the budget: the model + // lives on the card, so system RAM is not what bounds it. + It("prefers VRAM when the GPU reports it", func() { + const vram = uint64(24) << 30 + + Expect(availableModelMemory(stateWithCapability("nvidia-cuda-12", vram))).To(Equal(vram)) + }) + + // With no GPU at all the model lives in system RAM. + It("uses system RAM when no GPU is detected", func() { + ram := hostRAM() + + Expect(availableModelMemory(stateWithCapability("default", 0))).To(Equal(ram)) + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index 18a314dfc..726fe07b8 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -287,16 +287,23 @@ const noGPUDetected = "default" // availableModelMemory reports how much memory a model may occupy on this host. // -// With a usable GPU the model lives in VRAM. Without one it lives in system +// With a discrete GPU the model lives in VRAM. Otherwise it lives in system // RAM, read through xsysinfo because that path is cgroup-aware: under // Kubernetes the container's limit, not the node's physical RAM, is what the // model actually gets. // +// A detected GPU whose VRAM reads as zero falls back to RAM rather than to +// zero. That is the normal shape of a unified-memory host, not an error: +// arm64 macs report the metal capability unconditionally, yet have no discrete +// VRAM pool for TotalAvailableVRAM to find, so the model's real budget is +// system RAM. Returning the zero here would strand every Mac on the base +// build no matter how much memory it has. +// // An unreadable RAM figure yields 0, which drops every variant with a known // requirement and installs the base. That is the safe direction: an unknown // host should not be talked into a larger download. func availableModelMemory(systemState *system.SystemState) uint64 { - if systemState.DetectedCapability() != noGPUDetected { + if systemState.DetectedCapability() != noGPUDetected && systemState.VRAM > 0 { return systemState.VRAM } ram, err := xsysinfo.GetSystemRAMInfo() diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 1c66967f2..b02ae336d 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -158,8 +158,10 @@ quantizations, or the same weights served by a different engine. Such an entry carries a `variants` list, and installing it normally lets LocalAI choose: - variants whose backend cannot run on this machine are dropped; -- variants that do not fit the available memory (VRAM on a GPU host, otherwise - system RAM) are dropped; +- variants that do not fit the available memory are dropped. That budget is + VRAM on a discrete-GPU host, and system RAM otherwise — including on + unified-memory machines such as Apple Silicon, where the GPU shares system + RAM and reports no separate VRAM pool; - the entry's own build is never dropped. It competes with whatever survived rather than waiting for everything else to fail, so an entry that is itself the largest build that fits keeps its own payload;