diff --git a/docs/content/features/GPU-acceleration.md b/docs/content/features/GPU-acceleration.md index fb939a47a..085617c24 100644 --- a/docs/content/features/GPU-acceleration.md +++ b/docs/content/features/GPU-acceleration.md @@ -458,6 +458,8 @@ watch -n1 rocm-smi rocm-smi --showallinfo ``` +LocalAI reads AMD GPU memory for `/api/resources` and the model fit check from `rocm-smi`. ROCm 7 and later images no longer ship `rocm-smi`; there LocalAI reads the same figures from the amdgpu driver in sysfs (`/sys/class/drm/card*/device/mem_info_*`). On APUs with a small VRAM carve-out and a large GTT pool (for example Strix Halo), the GTT pool counts toward GPU memory in both cases. + ### Intel ```bash diff --git a/pkg/xsysinfo/amd_sysfs_meminfo_test.go b/pkg/xsysinfo/amd_sysfs_meminfo_test.go new file mode 100644 index 000000000..5e75a3781 --- /dev/null +++ b/pkg/xsysinfo/amd_sysfs_meminfo_test.go @@ -0,0 +1,87 @@ +package xsysinfo + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("readAMDSysfsGPUMemory", func() { + var root string + + BeforeEach(func() { + root = filepath.Join(GinkgoT().TempDir(), "class", "drm") + Expect(os.MkdirAll(root, 0o755)).To(Succeed()) + }) + + // Values read inside a ROCm 7+ container on Strix Halo (gfx1151), + // where rocm-smi is no longer shipped. + strixHalo := map[string]string{ + "vendor": "0x1002\n", + "mem_info_vram_total": "536870912\n", + "mem_info_vram_used": "240300032\n", + "mem_info_gtt_total": "128849018880\n", + "mem_info_gtt_used": "33055911936\n", + } + + It("reports a shared-memory APU with GTT folded into totals", func() { + writeSysfsCard(root, "card0", "0000:c5:00.0", strixHalo) + + got := readAMDSysfsGPUMemory(root) + Expect(got).To(HaveLen(1)) + Expect(got[0].Vendor).To(Equal(VendorAMD)) + Expect(got[0].Index).To(Equal(0)) + Expect(got[0].TotalVRAM).To(Equal(uint64(536870912 + 128849018880))) + Expect(got[0].UsedVRAM).To(Equal(uint64(240300032 + 33055911936))) + Expect(got[0].FreeVRAM).To(Equal(got[0].TotalVRAM - got[0].UsedVRAM)) + Expect(got[0].UsagePercent).To(BeNumerically(">", 0)) + }) + + It("keeps a discrete card on VRAM only", func() { + writeSysfsCard(root, "card1", "0000:03:00.0", map[string]string{ + "vendor": "0x1002\n", + "mem_info_vram_total": "25769803776\n", + "mem_info_vram_used": "1073741824\n", + "mem_info_gtt_total": "34359738368\n", + "mem_info_gtt_used": "1048576\n", + }) + + got := readAMDSysfsGPUMemory(root) + Expect(got).To(HaveLen(1)) + Expect(got[0].Index).To(Equal(1)) + Expect(got[0].TotalVRAM).To(Equal(uint64(25769803776))) + Expect(got[0].UsedVRAM).To(Equal(uint64(1073741824))) + }) + + It("ignores non-AMD cards and connector entries", func() { + writeSysfsCard(root, "card0", "0000:c5:00.0", strixHalo) + // A connector hanging off the same device must not count twice. + Expect(os.MkdirAll(filepath.Join(root, "card0-DP-1"), 0o755)).To(Succeed()) + Expect(os.Symlink(filepath.Join(root, "..", "devices", "pci", "0000:c5:00.0"), + filepath.Join(root, "card0-DP-1", "device"))).To(Succeed()) + // An NVIDIA card exposing the same attribute names is not ours. + nv := map[string]string{} + for k, v := range strixHalo { + nv[k] = v + } + nv["vendor"] = "0x10de\n" + writeSysfsCard(root, "card1", "0000:01:00.0", nv) + + got := readAMDSysfsGPUMemory(root) + Expect(got).To(HaveLen(1)) + Expect(got[0].Index).To(Equal(0)) + }) + + It("reports nothing for an AMD card without mem_info attributes", func() { + writeSysfsCard(root, "card0", "0000:c5:00.0", map[string]string{ + "vendor": "0x1002\n", + }) + Expect(readAMDSysfsGPUMemory(root)).To(BeEmpty()) + }) + + It("reports nothing when the DRM tree is missing", func() { + Expect(readAMDSysfsGPUMemory(filepath.Join(root, "nope"))).To(BeEmpty()) + }) +}) diff --git a/pkg/xsysinfo/gpu.go b/pkg/xsysinfo/gpu.go index e017096b2..ab167ab68 100644 --- a/pkg/xsysinfo/gpu.go +++ b/pkg/xsysinfo/gpu.go @@ -614,10 +614,20 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo { return gpus } -// getAMDGPUMemory queries AMD GPUs using rocm-smi. -// Prefers --showmeminfo all so APU GTT pools are visible (#12058); falls -// back to vram-only on older rocm-smi builds that reject "all". +// getAMDGPUMemory queries AMD GPUs using rocm-smi, falling back to the +// amdgpu sysfs attributes when rocm-smi is absent (ROCm 7+ images ship +// only amd-smi) or reports nothing. func getAMDGPUMemory() []GPUMemoryInfo { + if gpus := getAMDRocmSMIMemory(); len(gpus) > 0 { + return gpus + } + return readAMDSysfsGPUMemory(defaultSysfsDRMPath) +} + +// getAMDRocmSMIMemory prefers --showmeminfo all so APU GTT pools are +// visible (#12058); falls back to vram-only on older rocm-smi builds that +// reject "all". +func getAMDRocmSMIMemory() []GPUMemoryInfo { if _, err := exec.LookPath("rocm-smi"); err != nil { return nil } @@ -758,38 +768,45 @@ func parseRocmSMIMemInfoCSV(output string) []GPUMemoryInfo { gttTotal, gttUsed = scaleRocmMemIfMB(gttTotal, gttUsed) } - totalBytes, usedBytes := vramTotal, vramUsed - if amdSharedMemoryAPU(vramTotal, gttTotal) { - totalBytes = vramTotal + gttTotal - usedBytes = vramUsed + gttUsed - xlog.Debug("AMD shared-memory APU detected; folding GTT into VRAM totals", - "vram_total", vramTotal, "gtt_total", gttTotal, "combined_total", totalBytes) - } - - freeBytes := uint64(0) - if totalBytes > usedBytes { - freeBytes = totalBytes - usedBytes - } - - usagePercent := 0.0 - if totalBytes > 0 { - usagePercent = float64(usedBytes) / float64(totalBytes) * 100 - } - - gpus = append(gpus, GPUMemoryInfo{ - Index: parseAMDDeviceIndex(parts[0]), - Name: "AMD GPU", - Vendor: VendorAMD, - TotalVRAM: totalBytes, - UsedVRAM: usedBytes, - FreeVRAM: freeBytes, - UsagePercent: usagePercent, - }) + gpus = append(gpus, amdGPUMemoryInfo(parseAMDDeviceIndex(parts[0]), + vramTotal, vramUsed, gttTotal, gttUsed)) } return gpus } +// amdGPUMemoryInfo builds the report for one AMD GPU, folding GTT into +// the totals when the sizes match a shared-memory APU (#12058). +func amdGPUMemoryInfo(index int, vramTotal, vramUsed, gttTotal, gttUsed uint64) GPUMemoryInfo { + totalBytes, usedBytes := vramTotal, vramUsed + if amdSharedMemoryAPU(vramTotal, gttTotal) { + totalBytes = vramTotal + gttTotal + usedBytes = vramUsed + gttUsed + xlog.Debug("AMD shared-memory APU detected; folding GTT into VRAM totals", + "vram_total", vramTotal, "gtt_total", gttTotal, "combined_total", totalBytes) + } + + freeBytes := uint64(0) + if totalBytes > usedBytes { + freeBytes = totalBytes - usedBytes + } + + usagePercent := 0.0 + if totalBytes > 0 { + usagePercent = float64(usedBytes) / float64(totalBytes) * 100 + } + + return GPUMemoryInfo{ + Index: index, + Name: "AMD GPU", + Vendor: VendorAMD, + TotalVRAM: totalBytes, + UsedVRAM: usedBytes, + FreeVRAM: freeBytes, + UsagePercent: usagePercent, + } +} + func looksLikeRocmMemInfoHeader(line string) bool { lower := strings.ToLower(line) return strings.Contains(lower, "device") && strings.Contains(lower, "memory") diff --git a/pkg/xsysinfo/sysfsgpu.go b/pkg/xsysinfo/sysfsgpu.go index e32686a2a..170f5f716 100644 --- a/pkg/xsysinfo/sysfsgpu.go +++ b/pkg/xsysinfo/sysfsgpu.go @@ -200,3 +200,44 @@ func ghwHasVendor(cards []*gpu.GraphicsCard, vendor string) bool { } return false } + +// readAMDSysfsGPUMemory reports AMD GPU memory from the amdgpu driver's +// mem_info_* attributes under the DRM sysfs tree. +// +// It is the fallback for images without rocm-smi: ROCm 7+ images ship +// only amd-smi, so a working Strix Halo reported "No GPU found" and +// gpu_count 0 while inference ran on the GPU. The kernel exposes the +// same VRAM and GTT figures rocm-smi reads, so the shared-memory APU +// rule (#12058) applies unchanged. +func readAMDSysfsGPUMemory(root string) []GPUMemoryInfo { + var gpus []GPUMemoryInfo + for _, card := range scanSysfsGPUs(root) { + if card.Vendor != VendorAMD { + continue + } + deviceDir := filepath.Join(root, card.Card, "device") + vramTotal, ok := readSysfsDecUint(filepath.Join(deviceDir, "mem_info_vram_total")) + if !ok { + continue + } + vramUsed, _ := readSysfsDecUint(filepath.Join(deviceDir, "mem_info_vram_used")) + gttTotal, _ := readSysfsDecUint(filepath.Join(deviceDir, "mem_info_gtt_total")) + gttUsed, _ := readSysfsDecUint(filepath.Join(deviceDir, "mem_info_gtt_used")) + gpus = append(gpus, amdGPUMemoryInfo(parseAMDDeviceIndex(card.Card), + vramTotal, vramUsed, gttTotal, gttUsed)) + } + return gpus +} + +func readSysfsDecUint(path string) (uint64, bool) { + // #nosec G304 -- same provenance as readSysfsHexUint. + raw, err := os.ReadFile(path) + if err != nil { + return 0, false + } + value, err := strconv.ParseUint(strings.TrimSpace(string(raw)), 10, 64) + if err != nil { + return 0, false + } + return value, true +}