From d90f703bd949c883829a9782fdcba080e3627e96 Mon Sep 17 00:00:00 2001 From: leilei3167 Date: Mon, 5 Oct 2026 07:37:12 +0800 Subject: [PATCH] fix(xsysinfo): read process VRAM without proc children (#12482) * fix(xsysinfo): read process VRAM without proc children Kernels without CONFIG_PROC_CHILDREN have no task children file, so ProcessVRAM dropped every DRM reading. When that file is missing, walk child processes from /proc//stat ppid links instead. Other read errors still drop the reading. Fixes #12481 Signed-off-by: leilei3167 * docs(system): describe the proc children fallback Document VRAM reporting on kernels without CONFIG_PROC_CHILDREN. Assisted-by: Codex:GPT-6 --------- Signed-off-by: leilei3167 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- docs/content/reference/system-info.md | 4 + pkg/xsysinfo/process_vram_linux.go | 117 ++++++++++++++++++++---- pkg/xsysinfo/process_vram_linux_test.go | 13 +++ 3 files changed, 116 insertions(+), 18 deletions(-) diff --git a/docs/content/reference/system-info.md b/docs/content/reference/system-info.md index ed4de00ff..a57d2b223 100644 --- a/docs/content/reference/system-info.md +++ b/docs/content/reference/system-info.md @@ -40,6 +40,10 @@ to the backend, without separating weights, KV cache, and other allocations. See the [kernel DRM accounting specification](https://docs.kernel.org/gpu/drm-usage-stats.html) for these counters. +Kernels without `CONFIG_PROC_CHILDREN` are supported. When `/proc` does not +provide per-thread `children` files, LocalAI finds child processes through +the parent PIDs in `/proc//stat`. + The field is omitted when accounting is unavailable or incomplete. This includes external and distributed backends, macOS, proprietary NVIDIA drivers, primary DRM nodes (`/dev/dri/card*`), missing resident counters, diff --git a/pkg/xsysinfo/process_vram_linux.go b/pkg/xsysinfo/process_vram_linux.go index 3dd0a59a9..1bf6d2131 100644 --- a/pkg/xsysinfo/process_vram_linux.go +++ b/pkg/xsysinfo/process_vram_linux.go @@ -26,6 +26,8 @@ func processVRAM(procRoot string, pid int) (uint64, bool) { clients := map[string]uint64{} seen := map[int]bool{} pending := []int{pid} + // Filled once, when a kernel without CONFIG_PROC_CHILDREN is detected. + var childIndex map[int][]int for len(pending) > 0 { current := pending[len(pending)-1] pending = pending[:len(pending)-1] @@ -73,26 +75,11 @@ func processVRAM(procRoot string, pid int) (uint64, bool) { } // A worker may be spawned by any thread, not just the thread leader. - tasks, err := os.ReadDir(filepath.Join(base, "task")) - if err != nil || len(tasks) == 0 { + children, ok := directChildPIDs(procRoot, base, current, &childIndex) + if !ok { return 0, false } - for _, task := range tasks { - // #nosec G304 -- procRoot is /proc in production (a temp dir in tests); - // base adds an integer PID, and task.Name comes from os.ReadDir. - // The kernel supplies these path components, not request input. - data, err := os.ReadFile(filepath.Join(base, "task", task.Name(), "children")) - if err != nil { - return 0, false - } - for _, raw := range strings.Fields(string(data)) { - child, err := strconv.Atoi(raw) - if err != nil || child <= 0 { - return 0, false - } - pending = append(pending, child) - } - } + pending = append(pending, children...) } var total uint64 for _, used := range clients { @@ -104,6 +91,100 @@ func processVRAM(procRoot string, pid int) (uint64, bool) { return total, len(clients) > 0 } +// directChildPIDs lists processes forked by pid. Kernels without +// CONFIG_PROC_CHILDREN have no task//children file. That absence is not +// an incomplete tree, so the walk continues from /proc//stat ppid links. +// A task that disappears mid-read, or any other error, still fails the reading. +func directChildPIDs(procRoot, base string, pid int, childIndex *map[int][]int) ([]int, bool) { + tasks, err := os.ReadDir(filepath.Join(base, "task")) + if err != nil || len(tasks) == 0 { + return nil, false + } + if *childIndex != nil { + return (*childIndex)[pid], true + } + var children []int + for _, task := range tasks { + taskDir := filepath.Join(base, "task", task.Name()) + // #nosec G304 -- procRoot is /proc in production (a temp dir in tests); + // base adds an integer PID, and task.Name comes from os.ReadDir. + // The kernel supplies these path components, not request input. + data, err := os.ReadFile(filepath.Join(taskDir, "children")) + if err != nil { + if !os.IsNotExist(err) { + return nil, false + } + if _, statErr := os.Stat(taskDir); statErr != nil { + return nil, false + } + mapped, ok := childPIDsByPPID(procRoot) + if !ok { + return nil, false + } + *childIndex = mapped + return mapped[pid], true + } + for _, raw := range strings.Fields(string(data)) { + child, err := strconv.Atoi(raw) + if err != nil || child <= 0 { + return nil, false + } + children = append(children, child) + } + } + return children, true +} + +func childPIDsByPPID(procRoot string) (map[int][]int, bool) { + entries, err := os.ReadDir(procRoot) + if err != nil { + return nil, false + } + children := map[int][]int{} + for _, entry := range entries { + if !entry.IsDir() { + continue + } + pid, err := strconv.Atoi(entry.Name()) + if err != nil || pid <= 0 { + continue + } + // #nosec G304 -- procRoot is /proc in production (a temp dir in tests); + // entry.Name is a numeric directory from os.ReadDir. + data, err := os.ReadFile(filepath.Join(procRoot, entry.Name(), "stat")) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, false + } + ppid, ok := ppidFromStat(string(data)) + if !ok { + return nil, false + } + children[ppid] = append(children[ppid], pid) + } + return children, true +} + +// ppidFromStat reads field 4 of /proc//stat. The comm field is wrapped in +// parentheses and may contain spaces or ')'. +func ppidFromStat(data string) (int, bool) { + end := strings.LastIndex(data, ")") + if end < 0 || end+1 >= len(data) { + return 0, false + } + fields := strings.Fields(data[end+1:]) + if len(fields) < 2 { + return 0, false + } + ppid, err := strconv.Atoi(fields[1]) + if err != nil || ppid < 0 { + return 0, false + } + return ppid, true +} + func drmResidentClient(data []byte) (string, uint64, bool) { var client string var total uint64 diff --git a/pkg/xsysinfo/process_vram_linux_test.go b/pkg/xsysinfo/process_vram_linux_test.go index 4de1f6cdd..c9d3905eb 100644 --- a/pkg/xsysinfo/process_vram_linux_test.go +++ b/pkg/xsysinfo/process_vram_linux_test.go @@ -78,6 +78,19 @@ var _ = Describe("ProcessVRAM", func() { Expect(ok).To(BeFalse()) }) + It("sums resident memory when the kernel has no children file", func() { + Expect(os.Remove(filepath.Join(root, "100/task/100/children"))).To(Succeed()) + write(filepath.Join(root, "100/stat"), "100 (backend) S 1 0 0\n") + Expect(os.MkdirAll(filepath.Join(root, "200/fd"), 0750)).To(Succeed()) + Expect(os.MkdirAll(filepath.Join(root, "200/task/200"), 0750)).To(Succeed()) + write(filepath.Join(root, "200/stat"), "200 (gpu (worker)) S 100 0 0\n") + addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n") + addFD(200, 3, "renderD129", "drm-client-id: 8\ndrm-resident-vram0: 256 MiB\n") + used, ok := processVRAM(root, 100) + Expect(ok).To(BeTrue()) + Expect(used).To(Equal(uint64(384 * 1024 * 1024))) + }) + It("omits a partial reading if another DRM client lacks accounting", func() { addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n") addFD(100, 4, "renderD129", "drm-client-id: 8\n")