feat(system): report per-model DRM VRAM (#12026)

* feat(system): report per-model DRM VRAM

Expose optional resident device memory for local backend process trees.
Deduplicate DRM clients and omit unsupported or incomplete readings.
Document accounting limits and preserve a measured zero in JSON.

Closes #11970.

Assisted-by: Codex:gpt-6

* fix(system): document trusted procfs reads

Scope G304 annotations to paths built from the fixed procfs root,
integer process IDs, and kernel directory entries. These reads accept
no user-controlled path components.

Assisted-by: Codex:GPT-6 gosec

---------

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

pkg/xsysinfo/process_vram_linux.go

@@ -0,0 +1,165 @@
+//go:build linux
+
+// SPDX-License-Identifier: MIT
+package xsysinfo
+
+import (
+	"bufio"
+	"bytes"
+	"math"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+// ProcessVRAM reports device-local resident bytes accounted to a process tree
+// by DRM. Unsupported or incomplete accounting returns false, not a measured zero.
+func ProcessVRAM(pid int) (uint64, bool) {
+	return processVRAM("/proc", pid)
+}
+
+func processVRAM(procRoot string, pid int) (uint64, bool) {
+	if pid <= 0 {
+		return 0, false
+	}
+	clients := map[string]uint64{}
+	seen := map[int]bool{}
+	pending := []int{pid}
+	for len(pending) > 0 {
+		current := pending[len(pending)-1]
+		pending = pending[:len(pending)-1]
+		if seen[current] {
+			continue
+		}
+		seen[current] = true
+		base := filepath.Join(procRoot, strconv.Itoa(current))
+		fds, err := os.ReadDir(filepath.Join(base, "fd"))
+		if err != nil {
+			return 0, false
+		}
+		for _, fd := range fds {
+			target, err := os.Readlink(filepath.Join(base, "fd", fd.Name()))
+			if err != nil {
+				return 0, false
+			}
+			// A mixed DRM/NVIDIA tree cannot provide a complete DRM reading.
+			if strings.HasPrefix(target, "/dev/nvidia") {
+				return 0, false
+			}
+			if !strings.HasPrefix(target, "/dev/dri/render") {
+				// Primary nodes can also own allocations. Until their device
+				// identity is resolved, omitting them would undercount the tree.
+				if strings.HasPrefix(target, "/dev/dri/") {
+					return 0, false
+				}
+				continue
+			}
+			// #nosec G304 -- procRoot is /proc in production (a temp dir in tests);
+			// base adds an integer PID, and fd.Name comes from os.ReadDir.
+			// The kernel supplies these path components, not request input.
+			data, err := os.ReadFile(filepath.Join(base, "fdinfo", fd.Name()))
+			if err != nil {
+				return 0, false
+			}
+			client, used, ok := drmResidentClient(data)
+			if !ok {
+				return 0, false
+			}
+			key := target + ":" + client
+			// dup() and fork() can expose the same client more than once. The
+			// snapshot is not atomic; retain its largest observed reading.
+			clients[key] = max(clients[key], used)
+		}
+
+		// A worker may be spawned by any thread, not just the thread leader.
+		tasks, err := os.ReadDir(filepath.Join(base, "task"))
+		if err != nil || len(tasks) == 0 {
+			return 0, false
+		}
+		for _, task := range tasks {
+			// #nosec G304 -- procRoot is /proc in production (a temp dir in tests);
+			// base adds an integer PID, and task.Name comes from os.ReadDir.
+			// The kernel supplies these path components, not request input.
+			data, err := os.ReadFile(filepath.Join(base, "task", task.Name(), "children"))
+			if err != nil {
+				return 0, false
+			}
+			for _, raw := range strings.Fields(string(data)) {
+				child, err := strconv.Atoi(raw)
+				if err != nil || child <= 0 {
+					return 0, false
+				}
+				pending = append(pending, child)
+			}
+		}
+	}
+	var total uint64
+	for _, used := range clients {
+		if used > math.MaxUint64-total {
+			return 0, false
+		}
+		total += used
+	}
+	return total, len(clients) > 0
+}
+
+func drmResidentClient(data []byte) (string, uint64, bool) {
+	var client string
+	var total uint64
+	found := false
+	scanner := bufio.NewScanner(bytes.NewReader(data))
+	for scanner.Scan() {
+		key, value, ok := strings.Cut(scanner.Text(), ":")
+		if !ok {
+			continue
+		}
+		if key == "drm-client-id" {
+			id, err := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
+			if err != nil {
+				return "", 0, false
+			}
+			client = strconv.FormatUint(id, 10)
+		}
+		region, resident := strings.CutPrefix(key, "drm-resident-")
+		if !resident || !isVRAMRegion(region) {
+			continue
+		}
+		used, ok := drmResidentBytes(value)
+		if !ok || used > math.MaxUint64-total {
+			return "", 0, false
+		}
+		total += used
+		found = true
+	}
+	return client, total, scanner.Err() == nil && client != "" && found
+}
+
+func drmResidentBytes(value string) (uint64, bool) {
+	fields := strings.Fields(value)
+	if len(fields) == 0 || len(fields) > 2 {
+		return 0, false
+	}
+	n, err := strconv.ParseUint(fields[0], 10, 64)
+	if err != nil {
+		return 0, false
+	}
+	unit := uint64(1)
+	if len(fields) == 2 {
+		switch strings.ToLower(fields[1]) {
+		case "b":
+		case "kib":
+			unit = 1 << 10
+		case "mib":
+			unit = 1 << 20
+		case "gib":
+			unit = 1 << 30
+		default:
+			return 0, false
+		}
+	}
+	if n > math.MaxUint64/unit {
+		return 0, false
+	}
+	return n * unit, true
+}

pkg/xsysinfo/process_vram_linux_test.go

@@ -0,0 +1,105 @@
+//go:build linux
+
+// SPDX-License-Identifier: MIT
+package xsysinfo
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ProcessVRAM", func() {
+	var root string
+	write := func(path, contents string) {
+		Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+		Expect(os.WriteFile(path, []byte(contents), 0600)).To(Succeed())
+	}
+	addProcess := func(pid int, children string) {
+		base := filepath.Join(root, strconv.Itoa(pid))
+		Expect(os.MkdirAll(filepath.Join(base, "fd"), 0750)).To(Succeed())
+		write(filepath.Join(base, "task", strconv.Itoa(pid), "children"), children)
+	}
+	addFD := func(pid, fd int, render, info string) {
+		base := filepath.Join(root, strconv.Itoa(pid))
+		name := strconv.Itoa(fd)
+		Expect(os.Symlink("/dev/dri/"+render, filepath.Join(base, "fd", name))).To(Succeed())
+		write(filepath.Join(base, "fdinfo", name), info)
+	}
+	BeforeEach(func() {
+		var err error
+		root, err = os.MkdirTemp("", "process-vram-")
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(os.RemoveAll, root)
+		addProcess(100, "")
+	})
+
+	It("sums resident device memory across GPUs and child processes without duplicate clients", func() {
+		write(filepath.Join(root, "100/task/101/children"), "200")
+		addProcess(200, "")
+		info := "drm-client-id: 7\ndrm-total-local0: 900 MiB\ndrm-resident-local0: 128 MiB\ndrm-resident-system0: 4 GiB\n"
+		addFD(100, 3, "renderD128", info)
+		addFD(100, 4, "renderD128", info)
+		addFD(200, 3, "renderD128", info)
+		addFD(200, 4, "renderD129", "drm-client-id: 7\ndrm-resident-vram0: 256 MiB\n")
+		used, ok := processVRAM(root, 100)
+		Expect(ok).To(BeTrue())
+		Expect(used).To(Equal(uint64(384 * 1024 * 1024)))
+	})
+
+	It("distinguishes a measured zero from unavailable accounting", func() {
+		addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-local0: 0 B\n")
+		used, ok := processVRAM(root, 100)
+		Expect(ok).To(BeTrue())
+		Expect(used).To(BeZero())
+	})
+
+	DescribeTable("does not invent readings from unsupported or invalid accounting",
+		func(info string) {
+			addFD(100, 3, "renderD128", info)
+			_, ok := processVRAM(root, 100)
+			Expect(ok).To(BeFalse())
+		},
+		Entry("no resident keys", "drm-client-id: 7\ndrm-total-vram0: 128 MiB\n"),
+		Entry("host memory only", "drm-client-id: 7\ndrm-resident-system0: 128 MiB\n"),
+		Entry("no client identity", "drm-resident-vram0: 128 MiB\n"),
+		Entry("malformed size", "drm-client-id: 7\ndrm-resident-vram0: unknown KiB\n"),
+		Entry("unknown unit", "drm-client-id: 7\ndrm-resident-vram0: 128 widgets\n"),
+		Entry("overflow", "drm-client-id: 7\ndrm-resident-vram0: 18446744073709551615 GiB\n"),
+	)
+
+	It("omits a partial reading if a child cannot be inspected", func() {
+		addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n")
+		write(filepath.Join(root, "100/task/100/children"), "200")
+		_, ok := processVRAM(root, 100)
+		Expect(ok).To(BeFalse())
+	})
+
+	It("omits a partial reading if another DRM client lacks accounting", func() {
+		addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n")
+		addFD(100, 4, "renderD129", "drm-client-id: 8\n")
+		_, ok := processVRAM(root, 100)
+		Expect(ok).To(BeFalse())
+	})
+
+	DescribeTable("omits mixed readings with unsupported GPU descriptors",
+		func(target string) {
+			addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n")
+			Expect(os.Symlink(target, filepath.Join(root, "100/fd/4"))).To(Succeed())
+			_, ok := processVRAM(root, 100)
+			Expect(ok).To(BeFalse())
+		},
+		Entry("primary DRM node", "/dev/dri/card0"),
+		Entry("NVIDIA device", "/dev/nvidia0"),
+	)
+
+	It("returns unavailable for missing processes or no DRM descriptors", func() {
+		for _, pid := range []int{-1, 0, 100, 999} {
+			_, ok := processVRAM(root, pid)
+			Expect(ok).To(BeFalse())
+		}
+	})
+})

pkg/xsysinfo/process_vram_other.go

@@ -0,0 +1,9 @@
+//go:build !linux
+
+// SPDX-License-Identifier: MIT
+package xsysinfo
+
+// ProcessVRAM is unavailable on platforms without Linux DRM fdinfo accounting.
+func ProcessVRAM(pid int) (uint64, bool) {
+	return 0, false
+}