From 8f56e4e042d3e67890648bea63749271237262ef Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 23 Aug 2026 08:55:13 +0200 Subject: [PATCH] fix(vram): persist remote probe metadata (#11487) * fix(vram): persist remote probe metadata The startup warmer repeated remote size and GGUF metadata probes after every restart because both caches lived only in memory. Store successful HTTP probes for 24 hours so frequent restarts reuse the prior results. Bound the cache, reject invalid records, and purge it when gallery data changes. Local model files continue to bypass persistence. Assisted-by: Codex:gpt-5 * fix(vram): check temporary file cleanup The lint gate rejects the unchecked cleanup call in the persistent cache writer. Assisted-by: Codex:gpt-5.6 [golangci-lint] * fix(vram): make persistent cache optional Remote metadata probes can transfer enough data that operators need control over disk reuse and startup warming. Gallery autoload now gates both behaviors, and the runtime setting applies changes immediately. Assisted-by: Codex:gpt-5 * fix(ui): expose gallery startup pre-warm The existing gallery autoload setting also gates the startup metadata warmer. Name both effects in Settings so operators can find the requested boot control. Assisted-by: Codex:gpt-5 --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/application/startup.go | 19 +- core/cli/run.go | 2 + core/config/application_config.go | 6 + core/config/application_config_test.go | 10 + core/config/runtime_settings.go | 1 + core/config/runtime_settings_registry.go | 4 + core/config/runtime_settings_startup.go | 1 + core/gallery/gallery.go | 4 + core/http/endpoints/localai/settings.go | 9 + .../e2e/settings-backend-logging.spec.js | 28 ++ core/http/react-ui/src/pages/Settings.jsx | 5 +- docs/content/advanced/vram-management.md | 12 +- docs/content/features/runtime-settings.md | 3 +- docs/content/reference/cli-reference.md | 1 + pkg/vram/cache.go | 335 +++++++++++++++++- pkg/vram/cache_persistent_test.go | 228 ++++++++++++ 16 files changed, 656 insertions(+), 12 deletions(-) create mode 100644 pkg/vram/cache_persistent_test.go diff --git a/core/application/startup.go b/core/application/startup.go index 17144b7a5..66a813162 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -446,13 +446,20 @@ func New(opts ...config.AppOption) (*Application, error) { // Wire gallery generation counter into VRAM caches so they invalidate // when gallery data refreshes instead of using a fixed TTL. vram.SetGalleryGenerationFunc(gallery.GalleryGeneration) + if options.AutoloadGalleries { + if options.VRAMPersistentCache { + // Remote GGUF probes can transfer substantial metadata. Keep successful + // results across restarts so the startup warmer does not repeat that work. + vram.ConfigurePersistentCache(filepath.Join(options.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour) + } - // Fill those caches ahead of the first visitor. An estimate for an entry - // nobody has asked about yet costs a remote probe of its weight files, and - // the model gallery asks for one per row, so without this the first page - // spends seconds filling in its own sizes while somebody watches it. - // Non-blocking, and bounded: see DefaultEstimateWarmConfig. - gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv()) + // Fill those caches ahead of the first visitor. An estimate for an entry + // nobody has asked about yet costs a remote probe of its weight files, and + // the model gallery asks for one per row, so without this the first page + // spends seconds filling in its own sizes while somebody watches it. + // Non-blocking, and bounded: see DefaultEstimateWarmConfig. + gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv()) + } if options.ConfigFile != "" { if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil { diff --git a/core/cli/run.go b/core/cli/run.go index 4bcd80f2c..6b9b3e4dc 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -53,6 +53,7 @@ type RunCMD struct { BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"` Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models" default:"${galleries}"` AutoloadGalleries bool `env:"LOCALAI_AUTOLOAD_GALLERIES,AUTOLOAD_GALLERIES" group:"models" default:"true"` + VRAMPersistentCache bool `env:"LOCALAI_VRAM_PERSISTENT_CACHE,VRAM_PERSISTENT_CACHE" group:"models" default:"true" help:"Persist successful remote VRAM metadata probes across restarts"` AutoloadBackendGalleries bool `env:"LOCALAI_AUTOLOAD_BACKEND_GALLERIES,AUTOLOAD_BACKEND_GALLERIES" group:"backends" default:"true"` BackendImagesReleaseTag string `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"` BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"` @@ -302,6 +303,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { config.WithF16(r.F16), config.WithStringGalleries(r.Galleries), config.WithBackendGalleries(r.BackendGalleries), + config.WithVRAMPersistentCache(r.VRAMPersistentCache), config.WithCors(r.CORS), config.WithCorsAllowOrigins(r.CORSAllowOrigins), config.WithDisableCSRF(r.DisableCSRF), diff --git a/core/config/application_config.go b/core/config/application_config.go index e6668f882..b83e8fe58 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -125,6 +125,7 @@ type ApplicationConfig struct { ExternalGRPCBackends map[string]string AutoloadGalleries, AutoloadBackendGalleries bool + VRAMPersistentCache bool AutoUpgradeBackends bool PreferDevelopmentBackends bool @@ -284,6 +285,7 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig { // toggle can still turn it off (a persisted false wins - see // loadRuntimeSettingsFromFile). EnableBackendLogging: true, + VRAMPersistentCache: true, ArtifactDownloadConcurrency: modelartifacts.DefaultDownloadConcurrency, AgentJobRetentionDays: 30, // Default: 30 days LRUEvictionMaxRetries: 30, // Default: 30 retries @@ -596,6 +598,10 @@ func WithAutoUpgradeBackends(v bool) AppOption { return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v } } +func WithVRAMPersistentCache(v bool) AppOption { + return func(o *ApplicationConfig) { o.VRAMPersistentCache = v } +} + func WithRequireBackendIntegrity(v bool) AppOption { return func(o *ApplicationConfig) { o.RequireBackendIntegrity = v } } diff --git a/core/config/application_config_test.go b/core/config/application_config_test.go index 6860388e7..8d8a4b753 100644 --- a/core/config/application_config_test.go +++ b/core/config/application_config_test.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "time" . "github.com/onsi/ginkgo/v2" @@ -9,6 +10,15 @@ import ( var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Describe("ToRuntimeSettings", func() { + It("includes the persistent VRAM cache toggle", func() { + encoded, err := json.Marshal(NewApplicationConfig().ToRuntimeSettings()) + Expect(err).NotTo(HaveOccurred()) + + var settings map[string]any + Expect(json.Unmarshal(encoded, &settings)).To(Succeed()) + Expect(settings).To(HaveKeyWithValue("vram_persistent_cache", true)) + }) + It("should convert all fields correctly", func() { appConfig := &ApplicationConfig{ WatchDog: true, diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go index 6e4381d8c..bd495104a 100644 --- a/core/config/runtime_settings.go +++ b/core/config/runtime_settings.go @@ -59,6 +59,7 @@ type RuntimeSettings struct { BackendGalleries *[]Gallery `json:"backend_galleries,omitempty"` AutoloadGalleries *bool `json:"autoload_galleries,omitempty"` AutoloadBackendGalleries *bool `json:"autoload_backend_galleries,omitempty"` + VRAMPersistentCache *bool `json:"vram_persistent_cache,omitempty"` // API keys - No omitempty as we need to save empty arrays to clear keys ApiKeys *[]string `json:"api_keys"` diff --git a/core/config/runtime_settings_registry.go b/core/config/runtime_settings_registry.go index ce5774ab4..fda45e5bc 100644 --- a/core/config/runtime_settings_registry.go +++ b/core/config/runtime_settings_registry.go @@ -328,6 +328,10 @@ var runtimeSettingsFields = []fieldSpec{ func(s *RuntimeSettings) **bool { return &s.AutoloadBackendGalleries }, func(o *ApplicationConfig) bool { return o.AutoloadBackendGalleries }, func(o *ApplicationConfig, v bool) { o.AutoloadBackendGalleries = v }), + field("vram_persistent_cache", + func(s *RuntimeSettings) **bool { return &s.VRAMPersistentCache }, + func(o *ApplicationConfig) bool { return o.VRAMPersistentCache }, + func(o *ApplicationConfig, v bool) { o.VRAMPersistentCache = v }), // API keys: echoed for the UI, but the apply loops never touch them. // The settings endpoint and the file watcher own the env+runtime merge diff --git a/core/config/runtime_settings_startup.go b/core/config/runtime_settings_startup.go index 116d2a4f1..9808c877b 100644 --- a/core/config/runtime_settings_startup.go +++ b/core/config/runtime_settings_startup.go @@ -45,6 +45,7 @@ func DefaultRuntimeBaseline() *ApplicationConfig { o.BackendGalleries = mustGalleries(DefaultBackendGalleriesJSON) o.AutoloadGalleries = true o.AutoloadBackendGalleries = true + o.VRAMPersistentCache = true // core/cli/run.go injects WithMemoryReclaimer(enabled, threshold) // unconditionally, so the kong threshold default (0.95) reaches the // config even when the reclaimer flag is off - this overlay must match diff --git a/core/gallery/gallery.go b/core/gallery/gallery.go index 1d23dfbdb..04752b3d1 100644 --- a/core/gallery/gallery.go +++ b/core/gallery/gallery.go @@ -16,6 +16,7 @@ import ( "github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" + "github.com/mudler/LocalAI/pkg/vram" "github.com/mudler/LocalAI/pkg/xsync" "github.com/mudler/xlog" @@ -457,6 +458,9 @@ func triggerGalleryRefresh(galleries []config.Gallery, systemState *system.Syste galleryGeneration.Add(1) } availableModelsMu.Unlock() + if changed { + vram.InvalidatePersistentCache() + } }() } diff --git a/core/http/endpoints/localai/settings.go b/core/http/endpoints/localai/settings.go index 15c6f6d92..606fe49c5 100644 --- a/core/http/endpoints/localai/settings.go +++ b/core/http/endpoints/localai/settings.go @@ -4,6 +4,7 @@ import ( "encoding/json" "io" "net/http" + "path/filepath" "time" "github.com/labstack/echo/v4" @@ -12,6 +13,7 @@ import ( "github.com/mudler/LocalAI/core/http/endpoints/openresponses" "github.com/mudler/LocalAI/core/p2p" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/vram" "github.com/mudler/LocalAI/pkg/vrambudget" "github.com/mudler/xlog" ) @@ -185,6 +187,13 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc { // Apply settings using centralized method watchdogChanged := appConfig.ApplyRuntimeSettings(&settings) + if settings.VRAMPersistentCache != nil || settings.AutoloadGalleries != nil { + if appConfig.VRAMPersistentCache && appConfig.AutoloadGalleries { + vram.ConfigurePersistentCache(filepath.Join(appConfig.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour) + } else { + vram.DisablePersistentCache() + } + } // Handle API keys specially (merge with startup keys) if settings.ApiKeys != nil { diff --git a/core/http/react-ui/e2e/settings-backend-logging.spec.js b/core/http/react-ui/e2e/settings-backend-logging.spec.js index 7b5459d89..8b1d03256 100644 --- a/core/http/react-ui/e2e/settings-backend-logging.spec.js +++ b/core/http/react-ui/e2e/settings-backend-logging.spec.js @@ -18,6 +18,34 @@ test.describe('Settings - Backend Logging', () => { await expect(input).toHaveValue('4') }) + test('persistent VRAM cache can be toggled', async ({ page }) => { + const row = page.locator('.form-row', { hasText: 'Persist remote VRAM estimates' }) + await expect(row).toBeVisible() + + const checkbox = row.locator('input[type="checkbox"]') + const wasChecked = await checkbox.isChecked() + await checkbox.locator('..').click() + if (wasChecked) { + await expect(checkbox).not.toBeChecked() + } else { + await expect(checkbox).toBeChecked() + } + }) + + test('gallery startup loading and pre-warming can be toggled together', async ({ page }) => { + const row = page.locator('.form-row', { hasText: 'Load and pre-warm galleries on boot' }) + await expect(row).toBeVisible() + + const checkbox = row.locator('input[type="checkbox"]') + const wasChecked = await checkbox.isChecked() + await checkbox.locator('..').click() + if (wasChecked) { + await expect(checkbox).not.toBeChecked() + } else { + await expect(checkbox).toBeChecked() + } + }) + test('backend logging toggle can be toggled', async ({ page }) => { // Find the checkbox associated with backend logging const section = page.locator('div', { has: page.locator('text=Enable Backend Logging') }) diff --git a/core/http/react-ui/src/pages/Settings.jsx b/core/http/react-ui/src/pages/Settings.jsx index c1a95ad29..df8990ff1 100644 --- a/core/http/react-ui/src/pages/Settings.jsx +++ b/core/http/react-ui/src/pages/Settings.jsx @@ -482,12 +482,15 @@ export default function Settings() { Galleries
- + update('autoload_galleries', v)} /> update('autoload_backend_galleries', v)} /> + + update('vram_persistent_cache', v)} /> +