From f154bd990a720130959e02c8eeea4e571e27117b Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 27 Sep 2026 21:18:34 +0200 Subject: [PATCH] feat(system): report per-model DRM VRAM (#12026) * feat(system): report per-model DRM VRAM Expose optional resident device memory for local backend process trees. Deduplicate DRM clients and omit unsupported or incomplete readings. Document accounting limits and preserve a measured zero in JSON. Closes #11970. Assisted-by: Codex:gpt-6 * fix(system): document trusted procfs reads Scope G304 annotations to paths built from the fixed procfs root, integer process IDs, and kernel directory entries. These reads accept no user-controlled path components. Assisted-by: Codex:GPT-6 gosec --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/http/endpoints/localai/system.go | 6 + .../endpoints/localai/system_info_test.go | 49 ++++++ core/schema/localai.go | 3 + core/schema/system_info_test.go | 25 +++ docs/content/reference/system-info.md | 24 +++ pkg/xsysinfo/process_vram_linux.go | 165 ++++++++++++++++++ pkg/xsysinfo/process_vram_linux_test.go | 105 +++++++++++ pkg/xsysinfo/process_vram_other.go | 9 + swagger/docs.go | 4 + swagger/swagger.json | 4 + swagger/swagger.yaml | 5 + 11 files changed, 399 insertions(+) create mode 100644 core/http/endpoints/localai/system_info_test.go create mode 100644 core/schema/system_info_test.go create mode 100644 pkg/xsysinfo/process_vram_linux.go create mode 100644 pkg/xsysinfo/process_vram_linux_test.go create mode 100644 pkg/xsysinfo/process_vram_other.go diff --git a/core/http/endpoints/localai/system.go b/core/http/endpoints/localai/system.go index 996c9a781..9c4ba7503 100644 --- a/core/http/endpoints/localai/system.go +++ b/core/http/endpoints/localai/system.go @@ -8,6 +8,7 @@ import ( "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/services/monitoring" "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/xsysinfo" ) // SystemInformations returns the system informations @@ -42,6 +43,11 @@ func SystemInformations(cl *config.ModelConfigLoader, ml *model.ModelLoader, app entry.Process = proc } } + if pid, ok := localPID(m); ok { + if used, ok := xsysinfo.ProcessVRAM(int(pid)); ok { + entry.SizeVRAM = &used + } + } sysmodels = append(sysmodels, entry) } if sampler != nil { diff --git a/core/http/endpoints/localai/system_info_test.go b/core/http/endpoints/localai/system_info_test.go new file mode 100644 index 000000000..83f7daa07 --- /dev/null +++ b/core/http/endpoints/localai/system_info_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT +package localai_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/endpoints/localai" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + process "github.com/mudler/go-processmanager" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SystemInformations memory", func() { + It("keeps model metadata and omits VRAM for remote or stopped backends", func() { + path, err := os.MkdirTemp("", "system-info-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, path) + configFile := filepath.Join(path, "remote.yaml") + Expect(os.WriteFile(configFile, []byte("name: remote\nbackend: llama-cpp\n"), 0600)).To(Succeed()) + cl := config.NewModelConfigLoader(path) + Expect(cl.ReadModelConfig(configFile)).To(Succeed()) + ml := model.NewModelLoader(&system.SystemState{}) + store := model.NewInMemoryModelStore() + store.Set("remote", model.NewModel("remote", "worker:50051", nil)) + store.Set("stopped", model.NewModel("stopped", "", &process.Process{})) + ml.SetModelStore(store) + app := echo.New() + app.GET("/system", localai.SystemInformations(cl, ml, &config.ApplicationConfig{}, nil)) + rec := httptest.NewRecorder() + app.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/system", nil)) + Expect(rec.Code).To(Equal(http.StatusOK)) + var response struct { + Models []map[string]any `json:"loaded_models"` + } + Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed()) + Expect(response.Models).To(ConsistOf( + map[string]any{"id": "remote", "backend": "llama-cpp"}, + map[string]any{"id": "stopped"}, + )) + }) +}) diff --git a/core/schema/localai.go b/core/schema/localai.go index 7e5d5e314..dc99a1dbe 100644 --- a/core/schema/localai.go +++ b/core/schema/localai.go @@ -208,6 +208,9 @@ type SysInfoModel struct { // when the model has no local process (a distributed worker holds it) or // the process could not be read. Process *SysInfoProcess `json:"process,omitempty"` + // SizeVRAM is DRM-accounted resident device memory in bytes. Nil means + // the backend process tree has no complete supported reading. + SizeVRAM *uint64 `json:"size_vram,omitempty"` } // SysInfoProcess is a point-in-time reading of one backend process. diff --git a/core/schema/system_info_test.go b/core/schema/system_info_test.go new file mode 100644 index 000000000..79a1bdba1 --- /dev/null +++ b/core/schema/system_info_test.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MIT +package schema_test + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SysInfoModel memory", func() { + It("omits unavailable VRAM while preserving a measured zero", func() { + entry := schema.SysInfoModel{ID: "model"} + encoded, err := json.Marshal(entry) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(MatchJSON(`{"id":"model"}`)) + + zero := uint64(0) + entry.SizeVRAM = &zero + encoded, err = json.Marshal(entry) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(MatchJSON(`{"id":"model","size_vram":0}`)) + }) +}) diff --git a/docs/content/reference/system-info.md b/docs/content/reference/system-info.md index b825e4e06..ed4de00ff 100644 --- a/docs/content/reference/system-info.md +++ b/docs/content/reference/system-info.md @@ -28,6 +28,29 @@ Returns available backends and currently loaded models. | `loaded_models[].process.memory_percent` | `number` | `rss_bytes` as a percentage of host RAM | | `loaded_models[].process.cpu_percent` | `number` | Share of the whole host's CPU used since the previous call, 0-100. Omitted on the first call that sees the process, because there is no earlier reading to compare against | | `loaded_models[].process.started_at` | `string` | When the process started (RFC 3339) | +| `loaded_models[].size_vram` | `integer` | Optional DRM-accounted resident device memory, in bytes | + +### Per-model VRAM + +On Linux, `size_vram` reports resident device memory for the local backend +process and its child processes. LocalAI reads `drm-resident-local*` and +`drm-resident-vram*` from `/proc` and counts each DRM client once per GPU. +Host-memory regions are excluded. The reading includes buffers attributed +to the backend, without separating weights, KV cache, and other allocations. +See the [kernel DRM accounting specification](https://docs.kernel.org/gpu/drm-usage-stats.html) +for these counters. + +The field is omitted when accounting is unavailable or incomplete. This +includes external and distributed backends, macOS, proprietary NVIDIA +drivers, primary DRM nodes (`/dev/dri/card*`), missing resident counters, +and unreadable process information. +A present value of `0` means the supported counters report zero bytes. +Treat an absent field as unknown. + +This is a snapshot of driver accounting, not a memory reservation. Shared +buffers can appear in different clients' counters, and allocations can change +during collection. Do not treat the sum across models as exclusive physical +GPU usage. These readings do not replace capacity checks when scheduling work. ### Usage @@ -49,6 +72,7 @@ curl http://localhost:8080/system { "id": "my-llama-model", "backend": "llama-cpp", + "size_vram": 5368709120, "process": { "pid": 48213, "rss_bytes": 5368709120, diff --git a/pkg/xsysinfo/process_vram_linux.go b/pkg/xsysinfo/process_vram_linux.go new file mode 100644 index 000000000..3dd0a59a9 --- /dev/null +++ b/pkg/xsysinfo/process_vram_linux.go @@ -0,0 +1,165 @@ +//go:build linux + +// SPDX-License-Identifier: MIT +package xsysinfo + +import ( + "bufio" + "bytes" + "math" + "os" + "path/filepath" + "strconv" + "strings" +) + +// ProcessVRAM reports device-local resident bytes accounted to a process tree +// by DRM. Unsupported or incomplete accounting returns false, not a measured zero. +func ProcessVRAM(pid int) (uint64, bool) { + return processVRAM("/proc", pid) +} + +func processVRAM(procRoot string, pid int) (uint64, bool) { + if pid <= 0 { + return 0, false + } + clients := map[string]uint64{} + seen := map[int]bool{} + pending := []int{pid} + for len(pending) > 0 { + current := pending[len(pending)-1] + pending = pending[:len(pending)-1] + if seen[current] { + continue + } + seen[current] = true + base := filepath.Join(procRoot, strconv.Itoa(current)) + fds, err := os.ReadDir(filepath.Join(base, "fd")) + if err != nil { + return 0, false + } + for _, fd := range fds { + target, err := os.Readlink(filepath.Join(base, "fd", fd.Name())) + if err != nil { + return 0, false + } + // A mixed DRM/NVIDIA tree cannot provide a complete DRM reading. + if strings.HasPrefix(target, "/dev/nvidia") { + return 0, false + } + if !strings.HasPrefix(target, "/dev/dri/render") { + // Primary nodes can also own allocations. Until their device + // identity is resolved, omitting them would undercount the tree. + if strings.HasPrefix(target, "/dev/dri/") { + return 0, false + } + continue + } + // #nosec G304 -- procRoot is /proc in production (a temp dir in tests); + // base adds an integer PID, and fd.Name comes from os.ReadDir. + // The kernel supplies these path components, not request input. + data, err := os.ReadFile(filepath.Join(base, "fdinfo", fd.Name())) + if err != nil { + return 0, false + } + client, used, ok := drmResidentClient(data) + if !ok { + return 0, false + } + key := target + ":" + client + // dup() and fork() can expose the same client more than once. The + // snapshot is not atomic; retain its largest observed reading. + clients[key] = max(clients[key], used) + } + + // A worker may be spawned by any thread, not just the thread leader. + tasks, err := os.ReadDir(filepath.Join(base, "task")) + if err != nil || len(tasks) == 0 { + return 0, false + } + for _, task := range tasks { + // #nosec G304 -- procRoot is /proc in production (a temp dir in tests); + // base adds an integer PID, and task.Name comes from os.ReadDir. + // The kernel supplies these path components, not request input. + data, err := os.ReadFile(filepath.Join(base, "task", task.Name(), "children")) + if err != nil { + return 0, false + } + for _, raw := range strings.Fields(string(data)) { + child, err := strconv.Atoi(raw) + if err != nil || child <= 0 { + return 0, false + } + pending = append(pending, child) + } + } + } + var total uint64 + for _, used := range clients { + if used > math.MaxUint64-total { + return 0, false + } + total += used + } + return total, len(clients) > 0 +} + +func drmResidentClient(data []byte) (string, uint64, bool) { + var client string + var total uint64 + found := false + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + key, value, ok := strings.Cut(scanner.Text(), ":") + if !ok { + continue + } + if key == "drm-client-id" { + id, err := strconv.ParseUint(strings.TrimSpace(value), 10, 64) + if err != nil { + return "", 0, false + } + client = strconv.FormatUint(id, 10) + } + region, resident := strings.CutPrefix(key, "drm-resident-") + if !resident || !isVRAMRegion(region) { + continue + } + used, ok := drmResidentBytes(value) + if !ok || used > math.MaxUint64-total { + return "", 0, false + } + total += used + found = true + } + return client, total, scanner.Err() == nil && client != "" && found +} + +func drmResidentBytes(value string) (uint64, bool) { + fields := strings.Fields(value) + if len(fields) == 0 || len(fields) > 2 { + return 0, false + } + n, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + return 0, false + } + unit := uint64(1) + if len(fields) == 2 { + switch strings.ToLower(fields[1]) { + case "b": + case "kib": + unit = 1 << 10 + case "mib": + unit = 1 << 20 + case "gib": + unit = 1 << 30 + default: + return 0, false + } + } + if n > math.MaxUint64/unit { + return 0, false + } + return n * unit, true +} diff --git a/pkg/xsysinfo/process_vram_linux_test.go b/pkg/xsysinfo/process_vram_linux_test.go new file mode 100644 index 000000000..4de1f6cdd --- /dev/null +++ b/pkg/xsysinfo/process_vram_linux_test.go @@ -0,0 +1,105 @@ +//go:build linux + +// SPDX-License-Identifier: MIT +package xsysinfo + +import ( + "os" + "path/filepath" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ProcessVRAM", func() { + var root string + write := func(path, contents string) { + Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed()) + Expect(os.WriteFile(path, []byte(contents), 0600)).To(Succeed()) + } + addProcess := func(pid int, children string) { + base := filepath.Join(root, strconv.Itoa(pid)) + Expect(os.MkdirAll(filepath.Join(base, "fd"), 0750)).To(Succeed()) + write(filepath.Join(base, "task", strconv.Itoa(pid), "children"), children) + } + addFD := func(pid, fd int, render, info string) { + base := filepath.Join(root, strconv.Itoa(pid)) + name := strconv.Itoa(fd) + Expect(os.Symlink("/dev/dri/"+render, filepath.Join(base, "fd", name))).To(Succeed()) + write(filepath.Join(base, "fdinfo", name), info) + } + BeforeEach(func() { + var err error + root, err = os.MkdirTemp("", "process-vram-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, root) + addProcess(100, "") + }) + + It("sums resident device memory across GPUs and child processes without duplicate clients", func() { + write(filepath.Join(root, "100/task/101/children"), "200") + addProcess(200, "") + info := "drm-client-id: 7\ndrm-total-local0: 900 MiB\ndrm-resident-local0: 128 MiB\ndrm-resident-system0: 4 GiB\n" + addFD(100, 3, "renderD128", info) + addFD(100, 4, "renderD128", info) + addFD(200, 3, "renderD128", info) + addFD(200, 4, "renderD129", "drm-client-id: 7\ndrm-resident-vram0: 256 MiB\n") + used, ok := processVRAM(root, 100) + Expect(ok).To(BeTrue()) + Expect(used).To(Equal(uint64(384 * 1024 * 1024))) + }) + + It("distinguishes a measured zero from unavailable accounting", func() { + addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-local0: 0 B\n") + used, ok := processVRAM(root, 100) + Expect(ok).To(BeTrue()) + Expect(used).To(BeZero()) + }) + + DescribeTable("does not invent readings from unsupported or invalid accounting", + func(info string) { + addFD(100, 3, "renderD128", info) + _, ok := processVRAM(root, 100) + Expect(ok).To(BeFalse()) + }, + Entry("no resident keys", "drm-client-id: 7\ndrm-total-vram0: 128 MiB\n"), + Entry("host memory only", "drm-client-id: 7\ndrm-resident-system0: 128 MiB\n"), + Entry("no client identity", "drm-resident-vram0: 128 MiB\n"), + Entry("malformed size", "drm-client-id: 7\ndrm-resident-vram0: unknown KiB\n"), + Entry("unknown unit", "drm-client-id: 7\ndrm-resident-vram0: 128 widgets\n"), + Entry("overflow", "drm-client-id: 7\ndrm-resident-vram0: 18446744073709551615 GiB\n"), + ) + + It("omits a partial reading if a child cannot be inspected", func() { + addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n") + write(filepath.Join(root, "100/task/100/children"), "200") + _, ok := processVRAM(root, 100) + Expect(ok).To(BeFalse()) + }) + + It("omits a partial reading if another DRM client lacks accounting", func() { + addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n") + addFD(100, 4, "renderD129", "drm-client-id: 8\n") + _, ok := processVRAM(root, 100) + Expect(ok).To(BeFalse()) + }) + + DescribeTable("omits mixed readings with unsupported GPU descriptors", + func(target string) { + addFD(100, 3, "renderD128", "drm-client-id: 7\ndrm-resident-vram0: 128 MiB\n") + Expect(os.Symlink(target, filepath.Join(root, "100/fd/4"))).To(Succeed()) + _, ok := processVRAM(root, 100) + Expect(ok).To(BeFalse()) + }, + Entry("primary DRM node", "/dev/dri/card0"), + Entry("NVIDIA device", "/dev/nvidia0"), + ) + + It("returns unavailable for missing processes or no DRM descriptors", func() { + for _, pid := range []int{-1, 0, 100, 999} { + _, ok := processVRAM(root, pid) + Expect(ok).To(BeFalse()) + } + }) +}) diff --git a/pkg/xsysinfo/process_vram_other.go b/pkg/xsysinfo/process_vram_other.go new file mode 100644 index 000000000..06cc49186 --- /dev/null +++ b/pkg/xsysinfo/process_vram_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +// SPDX-License-Identifier: MIT +package xsysinfo + +// ProcessVRAM is unavailable on platforms without Linux DRM fdinfo accounting. +func ProcessVRAM(pid int) (uint64, bool) { + return 0, false +} diff --git a/swagger/docs.go b/swagger/docs.go index dc2fcf063..c447043ae 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -7810,6 +7810,10 @@ const docTemplate = `{ }, "id": { "type": "string" + }, + "size_vram": { + "description": "SizeVRAM is DRM-accounted resident device memory in bytes. Nil means\nthe backend process tree has no complete supported reading.", + "type": "integer" } } }, diff --git a/swagger/swagger.json b/swagger/swagger.json index a1e71cbdb..b4e49b347 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -7807,6 +7807,10 @@ }, "id": { "type": "string" + }, + "size_vram": { + "description": "SizeVRAM is DRM-accounted resident device memory in bytes. Nil means\nthe backend process tree has no complete supported reading.", + "type": "integer" } } }, diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 12de303a0..34fcfc8d7 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -2654,6 +2654,11 @@ definitions: type: string id: type: string + size_vram: + description: |- + SizeVRAM is DRM-accounted resident device memory in bytes. Nil means + the backend process tree has no complete supported reading. + type: integer type: object schema.SystemInformationResponse: properties: