From dce91ba7aa3496ddbe86467e517d0daabedf92cd Mon Sep 17 00:00:00 2001 From: mudler-agent Date: Mon, 19 Oct 2026 22:58:38 +0200 Subject: [PATCH] fix(backends): prefer installed development builds (#12782) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ci): capture and surface ginkgo errors in run-coverage.sh The coverage script ran ginkgo per test root with `|| fail=1`, which swallowed the exit code. Combined with `make --output-sync=target`, which buffers all output and flushes at the end, the actual ginkgo error was invisible in interleaved CI logs. Every test suite passed but the script still exited 1, and the error message was buried in 22 000 lines of buffered output. Each ginkgo invocation now captures combined stdout+stderr to a per-root log file. On failure the last 80 lines are printed to stderr so the error is visible in the CI log. On success the log is replayed to stdout so test output remains visible. The build flags (--tags, --coverpkg) are quoted correctly so values containing spaces (e.g. --tags="debug auth") are preserved through eval. Signed-off-by: Ettore Di Giacinto * fix(ci): create output dir before writing ginkgo logs The previous commit added per-root log files but forgot to create the output directory before the first ginkgo invocation. On CI the coverage/ dir did not exist yet, so the shell redirect to cover-pkg.out.log failed with "No such file or directory" before ginkgo even ran. Signed-off-by: Ettore Di Giacinto * test(core/http): add canvas preview route to auth coverage allowlist Commit 9ad78381d8 added GET /api/canvas/preview/:id as a public route (no auth — the random id is the capability, an iframe cannot send API key headers) but did not add it to expectedPublicRoutes in the route auth coverage test. The test flagged the route as an anonymous-access leak, failing the entire ./core ginkgo suite on every CI run since. The POST /api/canvas/preview endpoint is auth-gated (returns 401 to anonymous requests) and is intentionally absent from the allowlist. Signed-off-by: Ettore Di Giacinto * fix(backends): prefer installed development builds Prefer compatible runnable development builds for generic aliases when the preference is enabled. Keep concrete names and disabled selection unchanged. Pass worker-local configuration into resolver state before registration. Synchronize saved frontend settings at startup and require a restart for resolver changes instead of switching registered paths during runtime. Add selection, root-shadowing, configuration, and restart-boundary tests. Document fallback behavior and worker-local activation. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto * fix(settings): isolate backend restart notices A development preference save must not restart the watchdog. Separate its server restart notification from the legacy reconfiguration signal. Preserve existing response fields and genuine watchdog updates. Exercise the real settings endpoint for both preference values, repeated saves, and mixed watchdog updates. Verify resolver state and registered paths remain unchanged until startup. Attribution correction for commit 85c46d2fac789c9f02b47a3d0b79fd9d7e7ba9fc: its Assisted-by: Codex:gpt-5 trailer is incorrect. The actual agent was nib using model gpt-6.1-sol. Preserve that commit as task history. Assisted-by: nib:gpt-6.1-sol Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .../development_preference_test.go | 74 ++++++++++++++++ core/application/startup.go | 10 ++- core/cli/development_preference_test.go | 27 ++++++ core/cli/run.go | 2 +- core/config/application_config.go | 5 +- core/config/runtime_settings.go | 2 +- core/config/runtime_settings_registry.go | 5 +- core/gallery/backends.go | 21 +++++ core/gallery/backends_test.go | 84 +++++++++++++++++++ core/http/endpoints/localai/settings.go | 10 ++- core/http/endpoints/localai/settings_test.go | 61 ++++++++++++++ core/schema/localai.go | 7 +- core/services/worker/config.go | 13 +-- .../worker/development_preference_test.go | 62 ++++++++++++++ core/services/worker/worker.go | 17 ++-- docs/content/features/backends.md | 43 ++++++++++ pkg/system/state.go | 4 +- 17 files changed, 422 insertions(+), 25 deletions(-) create mode 100644 core/application/development_preference_test.go create mode 100644 core/cli/development_preference_test.go create mode 100644 core/services/worker/development_preference_test.go diff --git a/core/application/development_preference_test.go b/core/application/development_preference_test.go new file mode 100644 index 000000000..4e6bb0190 --- /dev/null +++ b/core/application/development_preference_test.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("development backend preference", func() { + BeforeEach(func() { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") + }) + installedState := func() *system.SystemState { + root := GinkgoT().TempDir() + for _, name := range []string{"cpu-example-engine", "cpu-example-engine-development"} { + dir := filepath.Join(root, name) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + data, err := json.Marshal(gallery.BackendMetadata{Name: name, Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), nil, 0644)).To(Succeed()) + } + st, err := system.GetSystemState(system.WithBackendPath(root), system.WithBackendSystemPath(GinkgoT().TempDir())) + Expect(err).NotTo(HaveOccurred()) + return st + } + It("synchronizes saved true at startup, including default-valued explicit false", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, "runtime_settings.json"), []byte(`{"prefer_development_backends":true}`), 0644)).To(Succeed()) + st := installedState() + cfg := config.NewApplicationConfig(config.WithSystemState(st), config.WithPreferDevelopmentBackends(false)) + cfg.DynamicConfigsDir = dir + loadStartupSettings(cfg) + Expect(cfg.PreferDevelopmentBackends).To(BeTrue()) + Expect(st.PreferDevelopmentBackends).To(BeTrue()) + ml := model.NewModelLoader(st) + Expect(gallery.RegisterBackends(st, ml)).To(Succeed()) + path := ml.GetExternalBackend("example-engine") + Expect(path).To(HaveSuffix("cpu-example-engine-development/run.sh")) + }) + It("preserves system-state-only true", func() { + st, err := system.GetSystemState(system.WithPreferDevelopmentBackends(true)) + Expect(err).NotTo(HaveOccurred()) + cfg := config.NewApplicationConfig(config.WithSystemState(st)) + loadStartupSettings(cfg) + Expect(st.PreferDevelopmentBackends).To(BeTrue()) + }) + It("does not request watchdog reconfiguration or hot-switch API or file-watcher state", func() { + st := installedState() + cfg := config.NewApplicationConfig(config.WithSystemState(st)) + ml := model.NewModelLoader(st) + Expect(gallery.RegisterBackends(st, ml)).To(Succeed()) + original := ml.GetExternalBackend("example-engine") + Expect(original).To(HaveSuffix("cpu-example-engine/run.sh")) + yes := true + Expect(cfg.ApplyRuntimeSettings(&config.RuntimeSettings{PreferDevelopmentBackends: &yes})).To(BeFalse()) + Expect(cfg.PreferDevelopmentBackends).To(BeTrue()) + Expect(st.PreferDevelopmentBackends).To(BeFalse()) + cfg.PreferDevelopmentBackends = false + Expect(readRuntimeSettingsJson(config.ApplicationConfig{})([]byte(`{"prefer_development_backends":true}`), cfg)).To(Succeed()) + Expect(cfg.PreferDevelopmentBackends).To(BeTrue()) + Expect(st.PreferDevelopmentBackends).To(BeFalse()) + current := ml.GetExternalBackend("example-engine") + Expect(current).To(Equal(original)) + }) +}) diff --git a/core/application/startup.go b/core/application/startup.go index 08b173213..b1be5f751 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -53,7 +53,7 @@ func New(opts ...config.AppOption) (*Application, error) { // were read) meant boot-loaded models saw pre-file defaults for one // full restart. Env/CLI values win over the file (see // ApplyRuntimeSettingsAtStartup). - loadRuntimeSettingsFromFile(options) + loadStartupSettings(options) // WithThreads no longer eagerly resolves 0 (so the settings merge can // tell "unset" from "env-set"); resolve the physical-core default now @@ -783,3 +783,11 @@ func (a *Application) registerFailoverMetrics() { } failover.RegisterMetrics(a.failoverManager, a.metricsService.Meter) } + +// loadStartupSettings freezes the resolver preference after the persisted +// settings merge, before installation and registration consume system state. +// Runtime API and file-watcher updates deliberately do not repeat this step. +func loadStartupSettings(options *config.ApplicationConfig) { + loadRuntimeSettingsFromFile(options) + options.SystemState.PreferDevelopmentBackends = options.PreferDevelopmentBackends || options.SystemState.PreferDevelopmentBackends +} diff --git a/core/cli/development_preference_test.go b/core/cli/development_preference_test.go new file mode 100644 index 000000000..067123b86 --- /dev/null +++ b/core/cli/development_preference_test.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT +package cli + +import ( + "os" + + "github.com/alecthomas/kong" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("development backend preference", func() { + DescribeTable("exposes the embedded distributed worker flag", func(args []string, want bool) { + for _, key := range []string{"LOCALAI_PREFER_DEV_BACKENDS", "PREFER_DEV_BACKENDS"} { + GinkgoT().Setenv(key, "false") + Expect(os.Unsetenv(key)).To(Succeed()) + } + cmd := struct { + Worker WorkerCMD `cmd:""` + }{} + parser, err := kong.New(&cmd, kong.Vars{"basepath": GinkgoT().TempDir(), "galleries": "[]", "backends": "[]"}) + Expect(err).NotTo(HaveOccurred()) + _, err = parser.Parse(append([]string{"worker", "--nats-url=nats://example.invalid:4222", "--register-to=http://example.invalid:8080"}, args...)) + Expect(err).NotTo(HaveOccurred()) + Expect(cmd.Worker.PreferDevelopmentBackends).To(Equal(want)) + }, Entry("default", []string{}, false), Entry("enabled", []string{"--prefer-development-backends"}, true), Entry("explicit false", []string{"--prefer-development-backends=false"}, false)) +}) diff --git a/core/cli/run.go b/core/cli/run.go index 905586967..77f2f9cce 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -69,7 +69,7 @@ type RunCMD struct { BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"` BackendDevSuffix string `env:"LOCALAI_BACKEND_DEV_SUFFIX,BACKEND_DEV_SUFFIX" help:"Development suffix for backend images" group:"backends" default:"development"` AutoUpgradeBackends bool `env:"LOCALAI_AUTO_UPGRADE_BACKENDS,AUTO_UPGRADE_BACKENDS" help:"Automatically upgrade backends when new versions are detected" group:"backends" default:"false"` - PreferDevelopmentBackends bool `env:"LOCALAI_PREFER_DEV_BACKENDS,PREFER_DEV_BACKENDS" help:"Prefer development backend versions (shows development backends by default in UI)" group:"backends" default:"false"` + PreferDevelopmentBackends bool `env:"LOCALAI_PREFER_DEV_BACKENDS,PREFER_DEV_BACKENDS" help:"Prefer compatible installed development backends for generic aliases and development downloads (requires restart)" group:"backends" default:"false"` PreloadModels string `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"` Models []string `env:"LOCALAI_MODELS,MODELS" help:"A List of model configuration URLs to load" group:"models"` PreloadModelsConfig string `env:"LOCALAI_PRELOAD_MODELS_CONFIG,PRELOAD_MODELS_CONFIG" help:"A List of models to apply at startup. Path to a YAML config file" group:"models"` diff --git a/core/config/application_config.go b/core/config/application_config.go index 1a6042fb5..74d60fb9f 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -1181,8 +1181,9 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings { // ApplyRuntimeSettings applies RuntimeSettings to ApplicationConfig (the // live POST /api/settings path). Only non-nil fields are applied. Returns -// true if restart-requiring settings changed (see each row's -// restartRequired flag in runtimeSettingsFields). +// true when legacy service-reconfiguration fields were accepted (see each +// row's restartRequired flag in runtimeSettingsFields). The settings endpoint +// consumes this as a watchdog reconfiguration signal, not a server restart. func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (requireRestart bool) { if settings == nil { return false diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go index bd495104a..1ec708dfb 100644 --- a/core/config/runtime_settings.go +++ b/core/config/runtime_settings.go @@ -21,7 +21,7 @@ type RuntimeSettings struct { SingleBackend *bool `json:"single_backend,omitempty"` // Deprecated: use MaxActiveBackends = 1 instead MaxActiveBackends *int `json:"max_active_backends,omitempty"` // Maximum number of active backends (0 = unlimited, 1 = single backend mode) AutoUpgradeBackends *bool `json:"auto_upgrade_backends,omitempty"` // Automatically upgrade backends when new versions are detected - PreferDevelopmentBackends *bool `json:"prefer_development_backends,omitempty"` // Prefer development backend versions by default in UI + PreferDevelopmentBackends *bool `json:"prefer_development_backends,omitempty"` // Display/download preference; installed alias resolver changes require restart // Memory Reclaimer settings (works with GPU if available, otherwise RAM) MemoryReclaimerEnabled *bool `json:"memory_reclaimer_enabled,omitempty"` // Enable memory threshold monitoring MemoryReclaimerThreshold *float64 `json:"memory_reclaimer_threshold,omitempty"` // Threshold 0.0-1.0 (e.g., 0.95 = 95%) diff --git a/core/config/runtime_settings_registry.go b/core/config/runtime_settings_registry.go index fda45e5bc..101733b4d 100644 --- a/core/config/runtime_settings_registry.go +++ b/core/config/runtime_settings_registry.go @@ -18,8 +18,9 @@ type fieldSpec struct { // jsonNames this spec owns; a single entry except for composite rows // (max_active_backends + its deprecated alias single_backend). jsonNames []string - // requiresRestart marks fields whose live change makes - // ApplyRuntimeSettings report a restart to the settings endpoint. + // requiresRestart is the legacy service-reconfiguration marker consumed + // by the settings endpoint as a watchdog restart signal. Server-only + // restart notifications must not use this marker. requiresRestart bool // fileAuthoritative: at startup/watch time the persisted value applies // unconditionally - either no env/CLI source exists (branding) or the diff --git a/core/gallery/backends.go b/core/gallery/backends.go index b2a16f997..6cb5be2bf 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -781,6 +781,27 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) slices.SortFunc(cands, func(a, b backendCandidate) int { return strings.Compare(a.name, b.name) }) + if systemState.PreferDevelopmentBackends { + _, _, suffix := getFallbackTagValues(systemState) + var development, stable []backendCandidate + for _, c := range cands { + if !isBackendRunnable(SystemBackend{RunFile: c.runFile}) || + !systemState.IsBackendCompatible(c.name, c.metadata.URI) { + continue + } + if strings.HasSuffix(c.name, "-"+suffix) { + development = append(development, c) + } else { + stable = append(stable, c) + } + } + // Filter only the enabled path. Partitioning preserves lexical order + // and lets development preference precede hardware token ranking. + cands = stable + if len(development) > 0 { + cands = development + } + } chosen := backendCandidate{} // Try preference tokens for _, t := range tokens { diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index ec455d582..f50865173 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -1579,3 +1579,87 @@ var _ = Describe("Gallery Backends", func() { }) }) }) + +var _ = Describe("installed development backend preference", func() { + BeforeEach(func() { GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") }) + DescribeTable("selects eligible installed aliases without redirecting concrete entries", + func(enabled bool, capability, suffix string, names []string, missing string, want string) { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", capability) + root := GinkgoT().TempDir() + for _, name := range names { + dir := filepath.Join(root, name) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + data, err := json.Marshal(BackendMetadata{Name: name, Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + if name != missing { + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), nil, 0644)).To(Succeed()) + } + } + st, err := system.GetSystemState(system.WithBackendPath(root), system.WithBackendSystemPath(GinkgoT().TempDir()), system.WithPreferDevelopmentBackends(enabled), system.WithBackendDevSuffix(suffix)) + Expect(err).NotTo(HaveOccurred()) + for range 3 { + found, err := ListSystemBackends(st) + Expect(err).NotTo(HaveOccurred()) + if want == "" { + Expect(found).NotTo(HaveKey("example-engine")) + } else { + Expect(found["example-engine"].RunFile).To(Equal(filepath.Join(root, want, "run.sh"))) + } + for _, name := range names { + if name != missing { + Expect(found[name].RunFile).To(Equal(filepath.Join(root, name, "run.sh"))) + } + } + } + }, + Entry("enabled coexistence", true, "default", "", []string{"cpu-example-engine", "cpu-example-engine-development"}, "", "cpu-example-engine-development"), + Entry("disabled coexistence", false, "default", "", []string{"cpu-example-engine-development", "cpu-example-engine"}, "", "cpu-example-engine"), + Entry("disabled development only", false, "default", "", []string{"cpu-example-engine-development"}, "", "cpu-example-engine-development"), + Entry("incompatible development", true, "nvidia-cuda-12", "", []string{"rocm-example-engine-development", "cuda12-example-engine"}, "", "cuda12-example-engine"), + Entry("development before hardware rank", true, "nvidia-cuda-12", "", []string{"cuda12-example-engine", "cpu-example-engine-development"}, "", "cpu-example-engine-development"), + Entry("hardware rank within development", true, "nvidia-cuda-12", "", []string{"cpu-example-engine-development", "cuda12-example-engine-development"}, "", "cuda12-example-engine-development"), + Entry("missing run file", true, "default", "", []string{"cpu-example-engine-development", "cpu-example-engine"}, "cpu-example-engine-development", "cpu-example-engine"), + Entry("no eligible candidate", true, "default", "", []string{"rocm-example-engine-development"}, "", ""), + Entry("custom exact suffix", true, "default", "preview", []string{"cpu-example-engine-preview-extra", "cpu-example-engine-preview", "cpu-example-engine-development"}, "", "cpu-example-engine-preview"), + Entry("reversed creation", true, "default", "", []string{"cpu-example-engine-development", "cpu-example-engine"}, "", "cpu-example-engine-development"), + Entry("disabled compatibility filter", true, "disable", "", []string{"rocm-example-engine-development", "cpu-example-engine"}, "", "rocm-example-engine-development"), + ) +}) + +var _ = Describe("installed development backend preference root guards", func() { + It("keeps managed family shadowing even when only the system development build is eligible", func() { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") + managed, sys := GinkgoT().TempDir(), GinkgoT().TempDir() + for _, fixture := range []struct{ root, name string }{{managed, "rocm-example-engine"}, {sys, "cpu-example-engine-development"}} { + dir := filepath.Join(fixture.root, fixture.name) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + data, err := json.Marshal(BackendMetadata{Name: fixture.name, Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), nil, 0644)).To(Succeed()) + } + st, err := system.GetSystemState(system.WithBackendPath(managed), system.WithBackendSystemPath(sys), system.WithPreferDevelopmentBackends(true)) + Expect(err).NotTo(HaveOccurred()) + found, err := ListSystemBackends(st) + Expect(err).NotTo(HaveOccurred()) + Expect(found).NotTo(HaveKey("example-engine")) + Expect(found).NotTo(HaveKey("cpu-example-engine-development")) + Expect(found["rocm-example-engine"].IsSystem).To(BeFalse()) + }) + It("rejects a directory run file without filtering the explicit concrete listing", func() { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") + root := GinkgoT().TempDir() + dir := filepath.Join(root, "cpu-example-engine-development") + Expect(os.MkdirAll(filepath.Join(dir, "run.sh"), 0755)).To(Succeed()) + data, err := json.Marshal(BackendMetadata{Name: "cpu-example-engine-development", Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + st, err := system.GetSystemState(system.WithBackendPath(root), system.WithBackendSystemPath(GinkgoT().TempDir()), system.WithPreferDevelopmentBackends(true)) + Expect(err).NotTo(HaveOccurred()) + found, err := ListSystemBackends(st) + Expect(err).NotTo(HaveOccurred()) + Expect(found).NotTo(HaveKey("example-engine")) + Expect(found).To(HaveKey("cpu-example-engine-development")) + }) +}) diff --git a/core/http/endpoints/localai/settings.go b/core/http/endpoints/localai/settings.go index c515c362d..485e6b86c 100644 --- a/core/http/endpoints/localai/settings.go +++ b/core/http/endpoints/localai/settings.go @@ -188,7 +188,12 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc { // Apply settings using centralized method prevGalleries, prevBackendGalleries := appConfig.Galleries, appConfig.BackendGalleries + // The legacy ApplyRuntimeSettings result drives live watchdog + // reconfiguration, not a server restart. Resolver preference changes + // take effect only at startup and must not cycle that unrelated service. + prevDevelopmentPreference := appConfig.PreferDevelopmentBackends watchdogChanged := appConfig.ApplyRuntimeSettings(&settings) + restartRequired := prevDevelopmentPreference != appConfig.PreferDevelopmentBackends gallery.ResetGalleryModelCacheIfChanged(prevGalleries, prevBackendGalleries, appConfig) if settings.VRAMPersistentCache != nil || settings.AutoloadGalleries != nil { if appConfig.VRAMPersistentCache && appConfig.AutoloadGalleries { @@ -334,8 +339,9 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc { } return c.JSON(http.StatusOK, schema.SettingsResponse{ - Success: true, - Message: "Settings updated successfully", + Success: true, + Message: "Settings updated successfully", + RestartRequired: restartRequired, }) } } diff --git a/core/http/endpoints/localai/settings_test.go b/core/http/endpoints/localai/settings_test.go index a02a61ef6..f5db0912d 100644 --- a/core/http/endpoints/localai/settings_test.go +++ b/core/http/endpoints/localai/settings_test.go @@ -180,6 +180,67 @@ var _ = Describe("Settings endpoints", func() { Expect(app.ModelLoader().GetWatchDog()).ToNot(BeNil(), "watchdog should be running after a cold enable, without waiting for a restart") }) + It("signals a development preference restart without cycling the watchdog or resolver", func() { + Expect(post(`{"watchdog_idle_enabled":true}`).Code).To(Equal(http.StatusOK)) + watchdog := app.ModelLoader().GetWatchDog() + Expect(watchdog).ToNot(BeNil()) + cfg := app.ApplicationConfig() + cfg.PreferDevelopmentBackends = false + cfg.SystemState.PreferDevelopmentBackends = false + app.ModelLoader().SetExternalBackend("example-engine", filepath.Join(tmp, "stable", "run.sh")) + paths := app.ModelLoader().GetAllExternalBackends(nil) + + rec := post(`{"prefer_development_backends":true}`) + Expect(rec.Code).To(Equal(http.StatusOK), rec.Body.String()) + // Check lifecycle isolation before the response so the regression also + // catches the old handler's unrelated watchdog restart. + Expect(app.ModelLoader().GetWatchDog()).To(BeIdenticalTo(watchdog)) + Expect(cfg.PreferDevelopmentBackends).To(BeTrue()) + Expect(cfg.SystemState.PreferDevelopmentBackends).To(BeFalse()) + Expect(app.ModelLoader().GetAllExternalBackends(nil)).To(Equal(paths)) + persisted, err := cfg.ReadPersistedSettings() + Expect(err).ToNot(HaveOccurred()) + Expect(persisted.PreferDevelopmentBackends).ToNot(BeNil()) + Expect(*persisted.PreferDevelopmentBackends).To(BeTrue()) + var response map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed()) + Expect(response).To(HaveKeyWithValue("success", true)) + Expect(response).To(HaveKeyWithValue("message", "Settings updated successfully")) + Expect(response).To(HaveKeyWithValue("restart_required", true)) + + // Re-saving the same value is not a new resolver change. + rec = post(`{"prefer_development_backends":true}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed()) + Expect(response).To(HaveKeyWithValue("restart_required", false)) + Expect(app.ModelLoader().GetWatchDog()).To(BeIdenticalTo(watchdog)) + + // A genuine watchdog update still replaces the live instance and does + // not request a server restart. + rec = post(`{"watchdog_interval":"3s"}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(app.ModelLoader().GetWatchDog()).ToNot(BeIdenticalTo(watchdog)) + Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed()) + Expect(response).To(HaveKeyWithValue("restart_required", false)) + + watchdog = app.ModelLoader().GetWatchDog() + rec = post(`{"prefer_development_backends":false}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed()) + Expect(response).To(HaveKeyWithValue("restart_required", true)) + Expect(cfg.PreferDevelopmentBackends).To(BeFalse()) + Expect(app.ModelLoader().GetWatchDog()).To(BeIdenticalTo(watchdog)) + + // A mixed save must independently perform both actions. + rec = post(`{"prefer_development_backends":true,"watchdog_interval":"4s"}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(json.Unmarshal(rec.Body.Bytes(), &response)).To(Succeed()) + Expect(response).To(HaveKeyWithValue("restart_required", true)) + Expect(app.ModelLoader().GetWatchDog()).ToNot(BeIdenticalTo(watchdog)) + Expect(cfg.PreferDevelopmentBackends).To(BeTrue()) + Expect(cfg.SystemState.PreferDevelopmentBackends).To(BeFalse()) + Expect(app.ModelLoader().GetAllExternalBackends(nil)).To(Equal(paths)) + }) // The UI lists from the cached model listing, which is keyed by nothing. // A gallery change through this endpoint (for example a tightened // verification policy) must drop it, or the UI keeps showing what the old diff --git a/core/schema/localai.go b/core/schema/localai.go index 47d911916..479cdfbc3 100644 --- a/core/schema/localai.go +++ b/core/schema/localai.go @@ -555,9 +555,10 @@ type KnownBackend struct { // SettingsResponse is the response type for settings API operations type SettingsResponse struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - Message string `json:"message,omitempty"` + Success bool `json:"success"` + Error string `json:"error,omitempty"` + Message string `json:"message,omitempty"` + RestartRequired bool `json:"restart_required"` } // RouterDecideRequest is the input for POST /api/router/decide — the diff --git a/core/services/worker/config.go b/core/services/worker/config.go index 8a63ad8e0..365d9f5b4 100644 --- a/core/services/worker/config.go +++ b/core/services/worker/config.go @@ -29,12 +29,13 @@ type Config struct { // concurrency. 0 = up to 65535. GRPCMaxPort int `env:"LOCALAI_GRPC_MAX_PORT" default:"0" help:"Highest port the worker may assign to a backend gRPC process. The range is [base port, this]; its width caps concurrent backends on this worker. 0 uses up to 65535." group:"server"` - BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends" group:"server"` - BackendsSystemPath string `env:"LOCALAI_BACKENDS_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends" group:"server"` - BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"server" default:"${backends}"` - Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of model galleries (used to resolve --prefetch-models on boot)" group:"server" default:"${galleries}"` - ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models" group:"server"` - RequireBackendIntegrity bool `env:"LOCALAI_REQUIRE_BACKEND_INTEGRITY,REQUIRE_BACKEND_INTEGRITY" help:"If true, reject backend installs without a configured signature verification policy (OCI URIs) or SHA256 (tarball/HTTP URIs)." group:"hardening" default:"false"` + PreferDevelopmentBackends bool `env:"LOCALAI_PREFER_DEV_BACKENDS,PREFER_DEV_BACKENDS" default:"false" help:"Prefer compatible installed development aliases and development downloads (worker-local; restart to apply)" group:"server"` + BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends" group:"server"` + BackendsSystemPath string `env:"LOCALAI_BACKENDS_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends" group:"server"` + BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"server" default:"${backends}"` + Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of model galleries (used to resolve --prefetch-models on boot)" group:"server" default:"${galleries}"` + ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models" group:"server"` + RequireBackendIntegrity bool `env:"LOCALAI_REQUIRE_BACKEND_INTEGRITY,REQUIRE_BACKEND_INTEGRITY" help:"If true, reject backend installs without a configured signature verification policy (OCI URIs) or SHA256 (tarball/HTTP URIs)." group:"hardening" default:"false"` // PrefetchModels lets a worker download gallery model artifacts (GGUFs, etc.) // from its own outbound internet at boot, instead of waiting for the master to diff --git a/core/services/worker/development_preference_test.go b/core/services/worker/development_preference_test.go new file mode 100644 index 000000000..736f7eb6a --- /dev/null +++ b/core/services/worker/development_preference_test.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT +package worker + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/alecthomas/kong" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("development backend preference", func() { + DescribeTable("parses worker configuration before registration", func(env, value string, args []string, want bool) { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") + for _, key := range []string{"LOCALAI_PREFER_DEV_BACKENDS", "PREFER_DEV_BACKENDS"} { + // Setenv registers restoration of the original value, then unset + // so Kong sees an absent bool rather than an invalid empty bool. + GinkgoT().Setenv(key, "false") + Expect(os.Unsetenv(key)).To(Succeed()) + } + if env != "" { + GinkgoT().Setenv(env, value) + } + cfg := Config{} + parser, err := kong.New(&cfg, kong.Vars{"basepath": GinkgoT().TempDir(), "galleries": "[]", "backends": "[]"}) + Expect(err).NotTo(HaveOccurred()) + _, err = parser.Parse(append([]string{"--nats-url=nats://example.invalid:4222", "--register-to=http://example.invalid:8080"}, args...)) + Expect(err).NotTo(HaveOccurred()) + cfg.BackendsPath = GinkgoT().TempDir() + cfg.BackendsSystemPath = GinkgoT().TempDir() + for _, name := range []string{"cpu-example-engine", "cpu-example-engine-development"} { + dir := filepath.Join(cfg.BackendsPath, name) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + data, err := json.Marshal(gallery.BackendMetadata{Name: name, Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), nil, 0644)).To(Succeed()) + } + st, err := cfg.systemState() + Expect(err).NotTo(HaveOccurred()) + Expect(st.PreferDevelopmentBackends).To(Equal(want)) + ml := model.NewModelLoader(st) + Expect(gallery.RegisterBackends(st, ml)).To(Succeed()) + supervisor := &backendSupervisor{cfg: &cfg, systemState: st, ml: ml} + chosen := "cpu-example-engine" + if want { + chosen += "-development" + } + Expect(supervisor.findBackend("example-engine")).To(Equal(filepath.Join(cfg.BackendsPath, chosen, "run.sh"))) + Expect(supervisor.findBackend("cpu-example-engine")).To(Equal(filepath.Join(cfg.BackendsPath, "cpu-example-engine", "run.sh"))) + }, + Entry("default", "", "", []string{}, false), + Entry("primary true", "LOCALAI_PREFER_DEV_BACKENDS", "true", []string{}, true), + Entry("legacy true", "PREFER_DEV_BACKENDS", "true", []string{}, true), + Entry("explicit false", "LOCALAI_PREFER_DEV_BACKENDS", "false", []string{}, false), + Entry("CLI true", "", "", []string{"--prefer-development-backends"}, true), + ) +}) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 79df4e884..4e7ba4e5c 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -39,12 +39,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { return fmt.Errorf("registration auth is required (LOCALAI_REGISTRATION_REQUIRE_AUTH or LOCALAI_DISTRIBUTED_REQUIRE_AUTH) but LOCALAI_REGISTRATION_TOKEN is empty — refusing to start an unauthenticated file-transfer server") } - systemState, err := system.GetSystemState( - system.WithModelPath(cfg.ModelsPath), - system.WithBackendPath(cfg.BackendsPath), - system.WithBackendSystemPath(cfg.BackendsSystemPath), - system.WithRequireBackendIntegrity(cfg.RequireBackendIntegrity), - ) + systemState, err := cfg.systemState() if err != nil { return fmt.Errorf("getting system state: %w", err) } @@ -300,3 +295,13 @@ func Run(ctx *cliContext.Context, cfg *Config) error { nodes.ShutdownFileTransferServer(httpServer) return runErr } + +func (cfg *Config) systemState() (*system.SystemState, error) { + return system.GetSystemState( + system.WithModelPath(cfg.ModelsPath), + system.WithBackendPath(cfg.BackendsPath), + system.WithBackendSystemPath(cfg.BackendsSystemPath), + system.WithRequireBackendIntegrity(cfg.RequireBackendIntegrity), + system.WithPreferDevelopmentBackends(cfg.PreferDevelopmentBackends), + ) +} diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index df466ba0d..b1c11fc3d 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -14,6 +14,49 @@ LocalAI ships **60+ backends** covering text generation, speech-to-text, text-to For the complete list of backends, the model families they support, and their acceleration targets, see the [Backend & Model Compatibility Table]({{%relref "reference/compatibility-table" %}}). The authoritative source is [`backend/index.yaml`](https://github.com/mudler/LocalAI/blob/master/backend/index.yaml), and the same catalog is browsable in the web UI under the **Backends** section. +## Prefer installed development backends + +Set `LOCALAI_PREFER_DEV_BACKENDS=true` (legacy alias: `PREFER_DEV_BACKENDS`) +or pass `--prefer-development-backends` to prefer development builds for generic +backend aliases. The default is `false`, which preserves existing selection, +including installations that contain only development builds. + +When enabled, LocalAI selects compatible installed builds with an existing, +non-directory run file. It prefers names ending in `-development` (or the +configured `LOCALAI_BACKEND_DEV_SUFFIX`). Within that group, existing hardware +ranking and lexical-name ordering still apply. If no eligible development build +exists, LocalAI selects an eligible non-development build. If neither group has +an eligible build, it does not register the alias. + +For example, generic `example-engine` selects `cpu-example-engine-development` +over `cpu-example-engine` when both are installed and compatible. A compatible +CPU development build also takes priority over a stable CUDA build. +An incompatible development build cannot take priority over a compatible stable +build. Compatibility uses LocalAI's existing hardware heuristics, including the +`LOCALAI_FORCE_META_BACKEND_CAPABILITY=disable` override; it does not guarantee +that native libraries can load. + +Explicit concrete names keep their original paths. Selecting +`cpu-example-engine` does not redirect to its development counterpart. +Installing a development image under a stable name does not rename that entry. +The preference also retains its existing development-first download behavior. + +The **Settings** value `prefer_development_backends` supplies the frontend's +preference at its next startup. POST `/api/settings` returns `restart_required: true` when this value changes, +while retaining the existing `success` and `message` response fields. Unchanged +values return `restart_required: false`. This notification does not reconfigure +the watchdog; genuine watchdog setting updates still apply live. Neither API +updates nor edits to `runtime_settings.json` change +registered paths or restart running models. The displayed preference can change +before the resolver adopts it. Startup uses the existing settings merge: +explicit default-valued CLI or environment values, including `false`, can be +indistinguishable from unset and do not necessarily override a saved `true`. + +For distributed `local-ai worker`, set the flag or environment variable on each +worker, then restart it. Frontend flags and Settings changes do not configure +remote workers. Legacy specialized worker subcommands do not expose this +worker-local option. + ## Managing Backends in the UI The **Operate → Runtime → Backends** page is the canonical home for the diff --git a/pkg/system/state.go b/pkg/system/state.go index e631c5c33..7139560f2 100644 --- a/pkg/system/state.go +++ b/pkg/system/state.go @@ -29,9 +29,11 @@ type SystemState struct { BackendImagesReleaseTag string BackendImagesBranchTag string BackendDevSuffix string - // PreferDevelopmentBackends installs the development image as the primary + // PreferDevelopmentBackends prefers compatible runnable installed development + // builds for generic aliases and installs the development image as the primary // backend URI (the released image becomes a fallback) rather than only using // development as a download fallback when the released image is missing. + // This resolver preference is fixed at startup, not hot-switched by settings. PreferDevelopmentBackends bool // RequireBackendIntegrity is the strict-integrity switch bound to