feat(system): report per-model DRM VRAM (#12026)

* feat(system): report per-model DRM VRAM

Expose optional resident device memory for local backend process trees.
Deduplicate DRM clients and omit unsupported or incomplete readings.
Document accounting limits and preserve a measured zero in JSON.

Closes #11970.

Assisted-by: Codex:gpt-6

* fix(system): document trusted procfs reads

Scope G304 annotations to paths built from the fixed procfs root,
integer process IDs, and kernel directory entries. These reads accept
no user-controlled path components.

Assisted-by: Codex:GPT-6 gosec

---------

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

core/http/endpoints/localai/system.go

@@ -8,6 +8,7 @@ import (
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/mudler/LocalAI/core/services/monitoring"
 	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/xsysinfo"
 )
 
 // SystemInformations returns the system informations
@@ -42,6 +43,11 @@ func SystemInformations(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
 					entry.Process = proc
 				}
 			}
+			if pid, ok := localPID(m); ok {
+				if used, ok := xsysinfo.ProcessVRAM(int(pid)); ok {
+					entry.SizeVRAM = &used
+				}
+			}
 			sysmodels = append(sysmodels, entry)
 		}
 		if sampler != nil {

core/http/endpoints/localai/system_info_test.go

@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: MIT
+package localai_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/http/endpoints/localai"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+	process "github.com/mudler/go-processmanager"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("SystemInformations memory", func() {
+	It("keeps model metadata and omits VRAM for remote or stopped backends", func() {
+		path, err := os.MkdirTemp("", "system-info-")
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(os.RemoveAll, path)
+		configFile := filepath.Join(path, "remote.yaml")
+		Expect(os.WriteFile(configFile, []byte("name: remote\nbackend: llama-cpp\n"), 0600)).To(Succeed())
+		cl := config.NewModelConfigLoader(path)
+		Expect(cl.ReadModelConfig(configFile)).To(Succeed())
+		ml := model.NewModelLoader(&system.SystemState{})
+		store := model.NewInMemoryModelStore()
+		store.Set("remote", model.NewModel("remote", "worker:50051", nil))
+		store.Set("stopped", model.NewModel("stopped", "", &process.Process{}))
+		ml.SetModelStore(store)
+		app := echo.New()
+		app.GET("/system", localai.SystemInformations(cl, ml, &config.ApplicationConfig{}, nil))
+		rec := httptest.NewRecorder()
+		app.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/system", nil))
+		Expect(rec.Code).To(Equal(http.StatusOK))
+		var response struct {
+			Models []map[string]any `json:"loaded_models"`
+		}
+		Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed())
+		Expect(response.Models).To(ConsistOf(
+			map[string]any{"id": "remote", "backend": "llama-cpp"},
+			map[string]any{"id": "stopped"},
+		))
+	})
+})

core/schema/localai.go

@@ -208,6 +208,9 @@ type SysInfoModel struct {
 	// when the model has no local process (a distributed worker holds it) or
 	// the process could not be read.
 	Process *SysInfoProcess `json:"process,omitempty"`
+	// SizeVRAM is DRM-accounted resident device memory in bytes. Nil means
+	// the backend process tree has no complete supported reading.
+	SizeVRAM *uint64 `json:"size_vram,omitempty"`
 }
 
 // SysInfoProcess is a point-in-time reading of one backend process.

core/schema/system_info_test.go

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: MIT
+package schema_test
+
+import (
+	"encoding/json"
+
+	"github.com/mudler/LocalAI/core/schema"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("SysInfoModel memory", func() {
+	It("omits unavailable VRAM while preserving a measured zero", func() {
+		entry := schema.SysInfoModel{ID: "model"}
+		encoded, err := json.Marshal(entry)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(string(encoded)).To(MatchJSON(`{"id":"model"}`))
+
+		zero := uint64(0)
+		entry.SizeVRAM = &zero
+		encoded, err = json.Marshal(entry)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(string(encoded)).To(MatchJSON(`{"id":"model","size_vram":0}`))
+	})
+})

docs/content/reference/system-info.md

@@ -28,6 +28,29 @@ Returns available backends and currently loaded models.
 | `loaded_models[].process.memory_percent` | `number` | `rss_bytes` as a percentage of host RAM |
 | `loaded_models[].process.cpu_percent` | `number` | Share of the whole host's CPU used since the previous call, 0-100. Omitted on the first call that sees the process, because there is no earlier reading to compare against |
 | `loaded_models[].process.started_at` | `string` | When the process started (RFC 3339) |
+| `loaded_models[].size_vram` | `integer` | Optional DRM-accounted resident device memory, in bytes |
+
+### Per-model VRAM
+
+On Linux, `size_vram` reports resident device memory for the local backend
+process and its child processes. LocalAI reads `drm-resident-local*` and
+`drm-resident-vram*` from `/proc` and counts each DRM client once per GPU.
+Host-memory regions are excluded. The reading includes buffers attributed
+to the backend, without separating weights, KV cache, and other allocations.
+See the [kernel DRM accounting specification](https://docs.kernel.org/gpu/drm-usage-stats.html)
+for these counters.
+
+The field is omitted when accounting is unavailable or incomplete. This
+includes external and distributed backends, macOS, proprietary NVIDIA
+drivers, primary DRM nodes (`/dev/dri/card*`), missing resident counters,
+and unreadable process information.
+A present value of `0` means the supported counters report zero bytes.
+Treat an absent field as unknown.
+
+This is a snapshot of driver accounting, not a memory reservation. Shared
+buffers can appear in different clients' counters, and allocations can change
+during collection. Do not treat the sum across models as exclusive physical
+GPU usage. These readings do not replace capacity checks when scheduling work.
 
 ### Usage
 
@@ -49,6 +72,7 @@ curl http://localhost:8080/system
     {
       "id": "my-llama-model",
       "backend": "llama-cpp",
+      "size_vram": 5368709120,
       "process": {
         "pid": 48213,
         "rss_bytes": 5368709120,

pkg/xsysinfo/process_vram_linux.go

@@ -0,0 +1,165 @@
+//go:build linux
+
+// SPDX-License-Identifier: MIT
+package xsysinfo
+
+import (
+	"bufio"
+	"bytes"
+	"math"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+// ProcessVRAM reports device-local resident bytes accounted to a process tree
+// by DRM. Unsupported or incomplete accounting returns false, not a measured zero.
+func ProcessVRAM(pid int) (uint64, bool) {
+	return processVRAM("/proc", pid)
+}
+
+func processVRAM(procRoot string, pid int) (uint64, bool) {
+	if pid <= 0 {
+		return 0, false
+	}
+	clients := map[string]uint64{}
+	seen := map[int]bool{}
+	pending := []int{pid}
+	for len(pending) > 0 {
+		current := pending[len(pending)-1]
+		pending = pending[:len(pending)-1]
+		if seen[current] {
+			continue
+		}
+		seen[current] = true
+		base := filepath.Join(procRoot, strconv.Itoa(current))
+		fds, err := os.ReadDir(filepath.Join(base, "fd"))
+		if err != nil {
+			return 0, false
+		}
+		for _, fd := range fds {
+			target, err := os.Readlink(filepath.Join(base, "fd", fd.Name()))
+			if err != nil {
+				return 0, false
+			}
+			// A mixed DRM/NVIDIA tree cannot provide a complete DRM reading.
+			if strings.HasPrefix(target, "/dev/nvidia") {
+				return 0, false
+			}
+			if !strings.HasPrefix(target, "/dev/dri/render") {
+				// Primary nodes can also own allocations. Until their device
+				// identity is resolved, omitting them would undercount the tree.
+				if strings.HasPrefix(target, "/dev/dri/") {
+					return 0, false
+				}
+				continue
+			}
+			// #nosec G304 -- procRoot is /proc in production (a temp dir in tests);
+			// base adds an integer PID, and fd.Name comes from os.ReadDir.
+			// The kernel supplies these path components, not request input.
+			data, err := os.ReadFile(filepath.Join(base, "fdinfo", fd.Name()))
+			if err != nil {
+				return 0, false
+			}
+			client, used, ok := drmResidentClient(data)
+			if !ok {
+				return 0, false
+			}
+			key := target + ":" + client
+			// dup() and fork() can expose the same client more than once. The
+			// snapshot is not atomic; retain its largest observed reading.
+			clients[key] = max(clients[key], used)
+		}
+
+		// A worker may be spawned by any thread, not just the thread leader.
+		tasks, err := os.ReadDir(filepath.Join(base, "task"))
+		if err != nil || len(tasks) == 0 {
+			return 0, false
+		}
+		for _, task := range tasks {
+			// #nosec G304 -- procRoot is /proc in production (a temp dir in tests);
+			// base adds an integer PID, and task.Name comes from os.ReadDir.
+			// The kernel supplies these path components, not request input.
+			data, err := os.ReadFile(filepath.Join(base, "task", task.Name(), "children"))
+			if err != nil {
+				return 0, false
+			}
+			for _, raw := range strings.Fields(string(data)) {
+				child, err := strconv.Atoi(raw)
+				if err != nil || child <= 0 {
+					return 0, false
+				}
+				pending = append(pending, child)
+			}
+		}
+	}
+	var total uint64
+	for _, used := range clients {
+		if used > math.MaxUint64-total {
+			return 0, false
+		}
+		total += used
+	}
+	return total, len(clients) > 0
+}
+
+func drmResidentClient(data []byte) (string, uint64, bool) {
+	var client string
+	var total uint64
+	found := false
+	scanner := bufio.NewScanner(bytes.NewReader(data))
+	for scanner.Scan() {
+		key, value, ok := strings.Cut(scanner.Text(), ":")
+		if !ok {
+			continue
+		}
+		if key == "drm-client-id" {
+			id, err := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
+			if err != nil {
+				return "", 0, false
+			}
+			client = strconv.FormatUint(id, 10)
+		}
+		region, resident := strings.CutPrefix(key, "drm-resident-")
+		if !resident || !isVRAMRegion(region) {
+			continue
+		}
+		used, ok := drmResidentBytes(value)
+		if !ok || used > math.MaxUint64-total {
+			return "", 0, false
+		}
+		total += used
+		found = true
+	}
+	return client, total, scanner.Err() == nil && client != "" && found
+}
+
+func drmResidentBytes(value string) (uint64, bool) {
+	fields := strings.Fields(value)
+	if len(fields) == 0 || len(fields) > 2 {
+		return 0, false
+	}
+	n, err := strconv.ParseUint(fields[0], 10, 64)
+	if err != nil {
+		return 0, false
+	}
+	unit := uint64(1)
+	if len(fields) == 2 {
+		switch strings.ToLower(fields[1]) {
+		case "b":
+		case "kib":
+			unit = 1 << 10
+		case "mib":
+			unit = 1 << 20
+		case "gib":
+			unit = 1 << 30
+		default:
+			return 0, false
+		}
+	}
+	if n > math.MaxUint64/unit {
+		return 0, false
+	}
+	return n * unit, true
+}

pkg/xsysinfo/process_vram_linux_test.go

@@ -0,0 +1,105 @@
+//go:build linux
+
+// SPDX-License-Identifier: MIT
+package xsysinfo
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ProcessVRAM", func() {
+	var root string
+	write := func(path, contents string) {
+		Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+		Expect(os.WriteFile(path, []byte(contents), 0600)).To(Succeed())
+	}
+	addProcess := func(pid int, children string) {
+		base := filepath.Join(root, strconv.Itoa(pid))
+		Expect(os.MkdirAll(filepath.Join(base, "fd"), 0750)).To(Succeed())
+		write(filepath.Join(base, "task", strconv.Itoa(pid), "children"), children)
+	}
+	addFD := func(pid, fd int, render, info string) {
+		base := filepath.Join(root, strconv.Itoa(pid))
+		name := strconv.Itoa(fd)
+		Expect(os.Symlink("/dev/dri/"+render, filepath.Join(base, "fd", name))).To(Succeed())
+		write(filepath.Join(base, "fdinfo", name), info)
+	}
+	BeforeEach(func() {
+		var err error
+		root, err = os.MkdirTemp("", "process-vram-")
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(os.RemoveAll, root)
+		addProcess(100, "")
+	})
+
+	It("sums resident device memory across GPUs and child processes without duplicate clients", func() {
+		write(filepath.Join(root, "100/task/101/children"), "200")
+		addProcess(200, "")
+		info := "drm-client-id: 7\ndrm-total-local0: 900 MiB\ndrm-resident-local0: 128 MiB\ndrm-resident-system0: 4 GiB\n"
+		addFD(100, 3, "renderD128", info)
+		addFD(100, 4, "renderD128", info)
+		addFD(200, 3, "renderD128", info)
+		addFD(200, 4, "renderD129", "drm-client-id: 7\ndrm-resident-vram0: 256 MiB\n")
+		used, ok := processVRAM(root, 100)
+		Expect(ok).To(BeTrue())
+		Expect(used).To(Equal(uint64(384 * 1024 * 1024)))
+	})
+
+	It("distinguishes a measured zero from unavailable accounting", func() {
+		addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-local0: 0 B\n")
+		used, ok := processVRAM(root, 100)
+		Expect(ok).To(BeTrue())
+		Expect(used).To(BeZero())
+	})
+
+	DescribeTable("does not invent readings from unsupported or invalid accounting",
+		func(info string) {
+			addFD(100, 3, "renderD128", info)
+			_, ok := processVRAM(root, 100)
+			Expect(ok).To(BeFalse())
+		},
+		Entry("no resident keys", "drm-client-id: 7\ndrm-total-vram0: 128 MiB\n"),
+		Entry("host memory only", "drm-client-id: 7\ndrm-resident-system0: 128 MiB\n"),
+		Entry("no client identity", "drm-resident-vram0: 128 MiB\n"),
+		Entry("malformed size", "drm-client-id: 7\ndrm-resident-vram0: unknown KiB\n"),
+		Entry("unknown unit", "drm-client-id: 7\ndrm-resident-vram0: 128 widgets\n"),
+		Entry("overflow", "drm-client-id: 7\ndrm-resident-vram0: 18446744073709551615 GiB\n"),
+	)
+
+	It("omits a partial reading if a child cannot be inspected", func() {
+		addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n")
+		write(filepath.Join(root, "100/task/100/children"), "200")
+		_, ok := processVRAM(root, 100)
+		Expect(ok).To(BeFalse())
+	})
+
+	It("omits a partial reading if another DRM client lacks accounting", func() {
+		addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n")
+		addFD(100, 4, "renderD129", "drm-client-id: 8\n")
+		_, ok := processVRAM(root, 100)
+		Expect(ok).To(BeFalse())
+	})
+
+	DescribeTable("omits mixed readings with unsupported GPU descriptors",
+		func(target string) {
+			addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n")
+			Expect(os.Symlink(target, filepath.Join(root, "100/fd/4"))).To(Succeed())
+			_, ok := processVRAM(root, 100)
+			Expect(ok).To(BeFalse())
+		},
+		Entry("primary DRM node", "/dev/dri/card0"),
+		Entry("NVIDIA device", "/dev/nvidia0"),
+	)
+
+	It("returns unavailable for missing processes or no DRM descriptors", func() {
+		for _, pid := range []int{-1, 0, 100, 999} {
+			_, ok := processVRAM(root, pid)
+			Expect(ok).To(BeFalse())
+		}
+	})
+})

pkg/xsysinfo/process_vram_other.go

@@ -0,0 +1,9 @@
+//go:build !linux
+
+// SPDX-License-Identifier: MIT
+package xsysinfo
+
+// ProcessVRAM is unavailable on platforms without Linux DRM fdinfo accounting.
+func ProcessVRAM(pid int) (uint64, bool) {
+	return 0, false
+}

swagger/docs.go

@@ -7810,6 +7810,10 @@ const docTemplate = `{
                 },
                 "id": {
                     "type": "string"
+                },
+                "size_vram": {
+                    "description": "SizeVRAM is DRM-accounted resident device memory in bytes. Nil means\nthe backend process tree has no complete supported reading.",
+                    "type": "integer"
                 }
             }
         },

swagger/swagger.json

@@ -7807,6 +7807,10 @@
                 },
                 "id": {
                     "type": "string"
+                },
+                "size_vram": {
+                    "description": "SizeVRAM is DRM-accounted resident device memory in bytes. Nil means\nthe backend process tree has no complete supported reading.",
+                    "type": "integer"
                 }
             }
         },

swagger/swagger.yaml

@@ -2654,6 +2654,11 @@ definitions:
         type: string
       id:
         type: string
+      size_vram:
+        description: |-
+          SizeVRAM is DRM-accounted resident device memory in bytes. Nil means
+          the backend process tree has no complete supported reading.
+        type: integer
     type: object
   schema.SystemInformationResponse:
     properties: