From 85c46d2fac789c9f02b47a3d0b79fd9d7e7ba9fc Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 19 Oct 2026 20:41:46 +0000 Subject: [PATCH] fix(backends): prefer installed development builds Prefer compatible runnable development builds for generic aliases when the preference is enabled. Keep concrete names and disabled selection unchanged. Pass worker-local configuration into resolver state before registration. Synchronize saved frontend settings at startup and require a restart for resolver changes instead of switching registered paths during runtime. Add selection, root-shadowing, configuration, and restart-boundary tests. Document fallback behavior and worker-local activation. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- .../development_preference_test.go | 74 ++++++++++++++++ core/application/startup.go | 10 ++- core/cli/development_preference_test.go | 27 ++++++ core/cli/run.go | 2 +- core/config/runtime_settings.go | 2 +- core/config/runtime_settings_registry.go | 3 +- core/gallery/backends.go | 21 +++++ core/gallery/backends_test.go | 84 +++++++++++++++++++ core/services/worker/config.go | 13 +-- .../worker/development_preference_test.go | 62 ++++++++++++++ core/services/worker/worker.go | 17 ++-- docs/content/features/backends.md | 40 +++++++++ pkg/system/state.go | 4 +- 13 files changed, 342 insertions(+), 17 deletions(-) create mode 100644 core/application/development_preference_test.go create mode 100644 core/cli/development_preference_test.go create mode 100644 core/services/worker/development_preference_test.go diff --git a/core/application/development_preference_test.go b/core/application/development_preference_test.go new file mode 100644 index 000000000..bffa03010 --- /dev/null +++ b/core/application/development_preference_test.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("development backend preference", func() { + BeforeEach(func() { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") + }) + installedState := func() *system.SystemState { + root := GinkgoT().TempDir() + for _, name := range []string{"cpu-example-engine", "cpu-example-engine-development"} { + dir := filepath.Join(root, name) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + data, err := json.Marshal(gallery.BackendMetadata{Name: name, Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), nil, 0644)).To(Succeed()) + } + st, err := system.GetSystemState(system.WithBackendPath(root), system.WithBackendSystemPath(GinkgoT().TempDir())) + Expect(err).NotTo(HaveOccurred()) + return st + } + It("synchronizes saved true at startup, including default-valued explicit false", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, "runtime_settings.json"), []byte(`{"prefer_development_backends":true}`), 0644)).To(Succeed()) + st := installedState() + cfg := config.NewApplicationConfig(config.WithSystemState(st), config.WithPreferDevelopmentBackends(false)) + cfg.DynamicConfigsDir = dir + loadStartupSettings(cfg) + Expect(cfg.PreferDevelopmentBackends).To(BeTrue()) + Expect(st.PreferDevelopmentBackends).To(BeTrue()) + ml := model.NewModelLoader(st) + Expect(gallery.RegisterBackends(st, ml)).To(Succeed()) + path := ml.GetExternalBackend("example-engine") + Expect(path).To(HaveSuffix("cpu-example-engine-development/run.sh")) + }) + It("preserves system-state-only true", func() { + st, err := system.GetSystemState(system.WithPreferDevelopmentBackends(true)) + Expect(err).NotTo(HaveOccurred()) + cfg := config.NewApplicationConfig(config.WithSystemState(st)) + loadStartupSettings(cfg) + Expect(st.PreferDevelopmentBackends).To(BeTrue()) + }) + It("reports restart without hot-switching API or file-watcher state", func() { + st := installedState() + cfg := config.NewApplicationConfig(config.WithSystemState(st)) + ml := model.NewModelLoader(st) + Expect(gallery.RegisterBackends(st, ml)).To(Succeed()) + original := ml.GetExternalBackend("example-engine") + Expect(original).To(HaveSuffix("cpu-example-engine/run.sh")) + yes := true + Expect(cfg.ApplyRuntimeSettings(&config.RuntimeSettings{PreferDevelopmentBackends: &yes})).To(BeTrue()) + Expect(cfg.PreferDevelopmentBackends).To(BeTrue()) + Expect(st.PreferDevelopmentBackends).To(BeFalse()) + cfg.PreferDevelopmentBackends = false + Expect(readRuntimeSettingsJson(config.ApplicationConfig{})([]byte(`{"prefer_development_backends":true}`), cfg)).To(Succeed()) + Expect(cfg.PreferDevelopmentBackends).To(BeTrue()) + Expect(st.PreferDevelopmentBackends).To(BeFalse()) + current := ml.GetExternalBackend("example-engine") + Expect(current).To(Equal(original)) + }) +}) diff --git a/core/application/startup.go b/core/application/startup.go index 08b173213..b1be5f751 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -53,7 +53,7 @@ func New(opts ...config.AppOption) (*Application, error) { // were read) meant boot-loaded models saw pre-file defaults for one // full restart. Env/CLI values win over the file (see // ApplyRuntimeSettingsAtStartup). - loadRuntimeSettingsFromFile(options) + loadStartupSettings(options) // WithThreads no longer eagerly resolves 0 (so the settings merge can // tell "unset" from "env-set"); resolve the physical-core default now @@ -783,3 +783,11 @@ func (a *Application) registerFailoverMetrics() { } failover.RegisterMetrics(a.failoverManager, a.metricsService.Meter) } + +// loadStartupSettings freezes the resolver preference after the persisted +// settings merge, before installation and registration consume system state. +// Runtime API and file-watcher updates deliberately do not repeat this step. +func loadStartupSettings(options *config.ApplicationConfig) { + loadRuntimeSettingsFromFile(options) + options.SystemState.PreferDevelopmentBackends = options.PreferDevelopmentBackends || options.SystemState.PreferDevelopmentBackends +} diff --git a/core/cli/development_preference_test.go b/core/cli/development_preference_test.go new file mode 100644 index 000000000..067123b86 --- /dev/null +++ b/core/cli/development_preference_test.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT +package cli + +import ( + "os" + + "github.com/alecthomas/kong" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("development backend preference", func() { + DescribeTable("exposes the embedded distributed worker flag", func(args []string, want bool) { + for _, key := range []string{"LOCALAI_PREFER_DEV_BACKENDS", "PREFER_DEV_BACKENDS"} { + GinkgoT().Setenv(key, "false") + Expect(os.Unsetenv(key)).To(Succeed()) + } + cmd := struct { + Worker WorkerCMD `cmd:""` + }{} + parser, err := kong.New(&cmd, kong.Vars{"basepath": GinkgoT().TempDir(), "galleries": "[]", "backends": "[]"}) + Expect(err).NotTo(HaveOccurred()) + _, err = parser.Parse(append([]string{"worker", "--nats-url=nats://example.invalid:4222", "--register-to=http://example.invalid:8080"}, args...)) + Expect(err).NotTo(HaveOccurred()) + Expect(cmd.Worker.PreferDevelopmentBackends).To(Equal(want)) + }, Entry("default", []string{}, false), Entry("enabled", []string{"--prefer-development-backends"}, true), Entry("explicit false", []string{"--prefer-development-backends=false"}, false)) +}) diff --git a/core/cli/run.go b/core/cli/run.go index 905586967..77f2f9cce 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -69,7 +69,7 @@ type RunCMD struct { BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"` BackendDevSuffix string `env:"LOCALAI_BACKEND_DEV_SUFFIX,BACKEND_DEV_SUFFIX" help:"Development suffix for backend images" group:"backends" default:"development"` AutoUpgradeBackends bool `env:"LOCALAI_AUTO_UPGRADE_BACKENDS,AUTO_UPGRADE_BACKENDS" help:"Automatically upgrade backends when new versions are detected" group:"backends" default:"false"` - PreferDevelopmentBackends bool `env:"LOCALAI_PREFER_DEV_BACKENDS,PREFER_DEV_BACKENDS" help:"Prefer development backend versions (shows development backends by default in UI)" group:"backends" default:"false"` + PreferDevelopmentBackends bool `env:"LOCALAI_PREFER_DEV_BACKENDS,PREFER_DEV_BACKENDS" help:"Prefer compatible installed development backends for generic aliases and development downloads (requires restart)" group:"backends" default:"false"` PreloadModels string `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"` Models []string `env:"LOCALAI_MODELS,MODELS" help:"A List of model configuration URLs to load" group:"models"` PreloadModelsConfig string `env:"LOCALAI_PRELOAD_MODELS_CONFIG,PRELOAD_MODELS_CONFIG" help:"A List of models to apply at startup. Path to a YAML config file" group:"models"` diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go index bd495104a..1ec708dfb 100644 --- a/core/config/runtime_settings.go +++ b/core/config/runtime_settings.go @@ -21,7 +21,7 @@ type RuntimeSettings struct { SingleBackend *bool `json:"single_backend,omitempty"` // Deprecated: use MaxActiveBackends = 1 instead MaxActiveBackends *int `json:"max_active_backends,omitempty"` // Maximum number of active backends (0 = unlimited, 1 = single backend mode) AutoUpgradeBackends *bool `json:"auto_upgrade_backends,omitempty"` // Automatically upgrade backends when new versions are detected - PreferDevelopmentBackends *bool `json:"prefer_development_backends,omitempty"` // Prefer development backend versions by default in UI + PreferDevelopmentBackends *bool `json:"prefer_development_backends,omitempty"` // Display/download preference; installed alias resolver changes require restart // Memory Reclaimer settings (works with GPU if available, otherwise RAM) MemoryReclaimerEnabled *bool `json:"memory_reclaimer_enabled,omitempty"` // Enable memory threshold monitoring MemoryReclaimerThreshold *float64 `json:"memory_reclaimer_threshold,omitempty"` // Threshold 0.0-1.0 (e.g., 0.95 = 95%) diff --git a/core/config/runtime_settings_registry.go b/core/config/runtime_settings_registry.go index fda45e5bc..240a1e7b4 100644 --- a/core/config/runtime_settings_registry.go +++ b/core/config/runtime_settings_registry.go @@ -169,7 +169,8 @@ var runtimeSettingsFields = []fieldSpec{ field("prefer_development_backends", func(s *RuntimeSettings) **bool { return &s.PreferDevelopmentBackends }, func(o *ApplicationConfig) bool { return o.PreferDevelopmentBackends }, - func(o *ApplicationConfig, v bool) { o.PreferDevelopmentBackends = v }), + func(o *ApplicationConfig, v bool) { o.PreferDevelopmentBackends = v }, + restartRequired()), // Memory reclaimer. Enabling it forces the watchdog master flag - that // cross-field invariant lives in the apply loops, not here. diff --git a/core/gallery/backends.go b/core/gallery/backends.go index b2a16f997..6cb5be2bf 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -781,6 +781,27 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) slices.SortFunc(cands, func(a, b backendCandidate) int { return strings.Compare(a.name, b.name) }) + if systemState.PreferDevelopmentBackends { + _, _, suffix := getFallbackTagValues(systemState) + var development, stable []backendCandidate + for _, c := range cands { + if !isBackendRunnable(SystemBackend{RunFile: c.runFile}) || + !systemState.IsBackendCompatible(c.name, c.metadata.URI) { + continue + } + if strings.HasSuffix(c.name, "-"+suffix) { + development = append(development, c) + } else { + stable = append(stable, c) + } + } + // Filter only the enabled path. Partitioning preserves lexical order + // and lets development preference precede hardware token ranking. + cands = stable + if len(development) > 0 { + cands = development + } + } chosen := backendCandidate{} // Try preference tokens for _, t := range tokens { diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index ec455d582..f50865173 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -1579,3 +1579,87 @@ var _ = Describe("Gallery Backends", func() { }) }) }) + +var _ = Describe("installed development backend preference", func() { + BeforeEach(func() { GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") }) + DescribeTable("selects eligible installed aliases without redirecting concrete entries", + func(enabled bool, capability, suffix string, names []string, missing string, want string) { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", capability) + root := GinkgoT().TempDir() + for _, name := range names { + dir := filepath.Join(root, name) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + data, err := json.Marshal(BackendMetadata{Name: name, Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + if name != missing { + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), nil, 0644)).To(Succeed()) + } + } + st, err := system.GetSystemState(system.WithBackendPath(root), system.WithBackendSystemPath(GinkgoT().TempDir()), system.WithPreferDevelopmentBackends(enabled), system.WithBackendDevSuffix(suffix)) + Expect(err).NotTo(HaveOccurred()) + for range 3 { + found, err := ListSystemBackends(st) + Expect(err).NotTo(HaveOccurred()) + if want == "" { + Expect(found).NotTo(HaveKey("example-engine")) + } else { + Expect(found["example-engine"].RunFile).To(Equal(filepath.Join(root, want, "run.sh"))) + } + for _, name := range names { + if name != missing { + Expect(found[name].RunFile).To(Equal(filepath.Join(root, name, "run.sh"))) + } + } + } + }, + Entry("enabled coexistence", true, "default", "", []string{"cpu-example-engine", "cpu-example-engine-development"}, "", "cpu-example-engine-development"), + Entry("disabled coexistence", false, "default", "", []string{"cpu-example-engine-development", "cpu-example-engine"}, "", "cpu-example-engine"), + Entry("disabled development only", false, "default", "", []string{"cpu-example-engine-development"}, "", "cpu-example-engine-development"), + Entry("incompatible development", true, "nvidia-cuda-12", "", []string{"rocm-example-engine-development", "cuda12-example-engine"}, "", "cuda12-example-engine"), + Entry("development before hardware rank", true, "nvidia-cuda-12", "", []string{"cuda12-example-engine", "cpu-example-engine-development"}, "", "cpu-example-engine-development"), + Entry("hardware rank within development", true, "nvidia-cuda-12", "", []string{"cpu-example-engine-development", "cuda12-example-engine-development"}, "", "cuda12-example-engine-development"), + Entry("missing run file", true, "default", "", []string{"cpu-example-engine-development", "cpu-example-engine"}, "cpu-example-engine-development", "cpu-example-engine"), + Entry("no eligible candidate", true, "default", "", []string{"rocm-example-engine-development"}, "", ""), + Entry("custom exact suffix", true, "default", "preview", []string{"cpu-example-engine-preview-extra", "cpu-example-engine-preview", "cpu-example-engine-development"}, "", "cpu-example-engine-preview"), + Entry("reversed creation", true, "default", "", []string{"cpu-example-engine-development", "cpu-example-engine"}, "", "cpu-example-engine-development"), + Entry("disabled compatibility filter", true, "disable", "", []string{"rocm-example-engine-development", "cpu-example-engine"}, "", "rocm-example-engine-development"), + ) +}) + +var _ = Describe("installed development backend preference root guards", func() { + It("keeps managed family shadowing even when only the system development build is eligible", func() { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") + managed, sys := GinkgoT().TempDir(), GinkgoT().TempDir() + for _, fixture := range []struct{ root, name string }{{managed, "rocm-example-engine"}, {sys, "cpu-example-engine-development"}} { + dir := filepath.Join(fixture.root, fixture.name) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + data, err := json.Marshal(BackendMetadata{Name: fixture.name, Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), nil, 0644)).To(Succeed()) + } + st, err := system.GetSystemState(system.WithBackendPath(managed), system.WithBackendSystemPath(sys), system.WithPreferDevelopmentBackends(true)) + Expect(err).NotTo(HaveOccurred()) + found, err := ListSystemBackends(st) + Expect(err).NotTo(HaveOccurred()) + Expect(found).NotTo(HaveKey("example-engine")) + Expect(found).NotTo(HaveKey("cpu-example-engine-development")) + Expect(found["rocm-example-engine"].IsSystem).To(BeFalse()) + }) + It("rejects a directory run file without filtering the explicit concrete listing", func() { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") + root := GinkgoT().TempDir() + dir := filepath.Join(root, "cpu-example-engine-development") + Expect(os.MkdirAll(filepath.Join(dir, "run.sh"), 0755)).To(Succeed()) + data, err := json.Marshal(BackendMetadata{Name: "cpu-example-engine-development", Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + st, err := system.GetSystemState(system.WithBackendPath(root), system.WithBackendSystemPath(GinkgoT().TempDir()), system.WithPreferDevelopmentBackends(true)) + Expect(err).NotTo(HaveOccurred()) + found, err := ListSystemBackends(st) + Expect(err).NotTo(HaveOccurred()) + Expect(found).NotTo(HaveKey("example-engine")) + Expect(found).To(HaveKey("cpu-example-engine-development")) + }) +}) diff --git a/core/services/worker/config.go b/core/services/worker/config.go index 8a63ad8e0..365d9f5b4 100644 --- a/core/services/worker/config.go +++ b/core/services/worker/config.go @@ -29,12 +29,13 @@ type Config struct { // concurrency. 0 = up to 65535. GRPCMaxPort int `env:"LOCALAI_GRPC_MAX_PORT" default:"0" help:"Highest port the worker may assign to a backend gRPC process. The range is [base port, this]; its width caps concurrent backends on this worker. 0 uses up to 65535." group:"server"` - BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends" group:"server"` - BackendsSystemPath string `env:"LOCALAI_BACKENDS_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends" group:"server"` - BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"server" default:"${backends}"` - Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of model galleries (used to resolve --prefetch-models on boot)" group:"server" default:"${galleries}"` - ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models" group:"server"` - RequireBackendIntegrity bool `env:"LOCALAI_REQUIRE_BACKEND_INTEGRITY,REQUIRE_BACKEND_INTEGRITY" help:"If true, reject backend installs without a configured signature verification policy (OCI URIs) or SHA256 (tarball/HTTP URIs)." group:"hardening" default:"false"` + PreferDevelopmentBackends bool `env:"LOCALAI_PREFER_DEV_BACKENDS,PREFER_DEV_BACKENDS" default:"false" help:"Prefer compatible installed development aliases and development downloads (worker-local; restart to apply)" group:"server"` + BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends" group:"server"` + BackendsSystemPath string `env:"LOCALAI_BACKENDS_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends" group:"server"` + BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"server" default:"${backends}"` + Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of model galleries (used to resolve --prefetch-models on boot)" group:"server" default:"${galleries}"` + ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models" group:"server"` + RequireBackendIntegrity bool `env:"LOCALAI_REQUIRE_BACKEND_INTEGRITY,REQUIRE_BACKEND_INTEGRITY" help:"If true, reject backend installs without a configured signature verification policy (OCI URIs) or SHA256 (tarball/HTTP URIs)." group:"hardening" default:"false"` // PrefetchModels lets a worker download gallery model artifacts (GGUFs, etc.) // from its own outbound internet at boot, instead of waiting for the master to diff --git a/core/services/worker/development_preference_test.go b/core/services/worker/development_preference_test.go new file mode 100644 index 000000000..736f7eb6a --- /dev/null +++ b/core/services/worker/development_preference_test.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT +package worker + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/alecthomas/kong" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("development backend preference", func() { + DescribeTable("parses worker configuration before registration", func(env, value string, args []string, want bool) { + GinkgoT().Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "default") + for _, key := range []string{"LOCALAI_PREFER_DEV_BACKENDS", "PREFER_DEV_BACKENDS"} { + // Setenv registers restoration of the original value, then unset + // so Kong sees an absent bool rather than an invalid empty bool. + GinkgoT().Setenv(key, "false") + Expect(os.Unsetenv(key)).To(Succeed()) + } + if env != "" { + GinkgoT().Setenv(env, value) + } + cfg := Config{} + parser, err := kong.New(&cfg, kong.Vars{"basepath": GinkgoT().TempDir(), "galleries": "[]", "backends": "[]"}) + Expect(err).NotTo(HaveOccurred()) + _, err = parser.Parse(append([]string{"--nats-url=nats://example.invalid:4222", "--register-to=http://example.invalid:8080"}, args...)) + Expect(err).NotTo(HaveOccurred()) + cfg.BackendsPath = GinkgoT().TempDir() + cfg.BackendsSystemPath = GinkgoT().TempDir() + for _, name := range []string{"cpu-example-engine", "cpu-example-engine-development"} { + dir := filepath.Join(cfg.BackendsPath, name) + Expect(os.MkdirAll(dir, 0755)).To(Succeed()) + data, err := json.Marshal(gallery.BackendMetadata{Name: name, Alias: "example-engine"}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), nil, 0644)).To(Succeed()) + } + st, err := cfg.systemState() + Expect(err).NotTo(HaveOccurred()) + Expect(st.PreferDevelopmentBackends).To(Equal(want)) + ml := model.NewModelLoader(st) + Expect(gallery.RegisterBackends(st, ml)).To(Succeed()) + supervisor := &backendSupervisor{cfg: &cfg, systemState: st, ml: ml} + chosen := "cpu-example-engine" + if want { + chosen += "-development" + } + Expect(supervisor.findBackend("example-engine")).To(Equal(filepath.Join(cfg.BackendsPath, chosen, "run.sh"))) + Expect(supervisor.findBackend("cpu-example-engine")).To(Equal(filepath.Join(cfg.BackendsPath, "cpu-example-engine", "run.sh"))) + }, + Entry("default", "", "", []string{}, false), + Entry("primary true", "LOCALAI_PREFER_DEV_BACKENDS", "true", []string{}, true), + Entry("legacy true", "PREFER_DEV_BACKENDS", "true", []string{}, true), + Entry("explicit false", "LOCALAI_PREFER_DEV_BACKENDS", "false", []string{}, false), + Entry("CLI true", "", "", []string{"--prefer-development-backends"}, true), + ) +}) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 79df4e884..4e7ba4e5c 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -39,12 +39,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { return fmt.Errorf("registration auth is required (LOCALAI_REGISTRATION_REQUIRE_AUTH or LOCALAI_DISTRIBUTED_REQUIRE_AUTH) but LOCALAI_REGISTRATION_TOKEN is empty — refusing to start an unauthenticated file-transfer server") } - systemState, err := system.GetSystemState( - system.WithModelPath(cfg.ModelsPath), - system.WithBackendPath(cfg.BackendsPath), - system.WithBackendSystemPath(cfg.BackendsSystemPath), - system.WithRequireBackendIntegrity(cfg.RequireBackendIntegrity), - ) + systemState, err := cfg.systemState() if err != nil { return fmt.Errorf("getting system state: %w", err) } @@ -300,3 +295,13 @@ func Run(ctx *cliContext.Context, cfg *Config) error { nodes.ShutdownFileTransferServer(httpServer) return runErr } + +func (cfg *Config) systemState() (*system.SystemState, error) { + return system.GetSystemState( + system.WithModelPath(cfg.ModelsPath), + system.WithBackendPath(cfg.BackendsPath), + system.WithBackendSystemPath(cfg.BackendsSystemPath), + system.WithRequireBackendIntegrity(cfg.RequireBackendIntegrity), + system.WithPreferDevelopmentBackends(cfg.PreferDevelopmentBackends), + ) +} diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index df466ba0d..25b1bc85c 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -14,6 +14,46 @@ LocalAI ships **60+ backends** covering text generation, speech-to-text, text-to For the complete list of backends, the model families they support, and their acceleration targets, see the [Backend & Model Compatibility Table]({{%relref "reference/compatibility-table" %}}). The authoritative source is [`backend/index.yaml`](https://github.com/mudler/LocalAI/blob/master/backend/index.yaml), and the same catalog is browsable in the web UI under the **Backends** section. +## Prefer installed development backends + +Set `LOCALAI_PREFER_DEV_BACKENDS=true` (legacy alias: `PREFER_DEV_BACKENDS`) +or pass `--prefer-development-backends` to prefer development builds for generic +backend aliases. The default is `false`, which preserves existing selection, +including installations that contain only development builds. + +When enabled, LocalAI selects compatible installed builds with an existing, +non-directory run file. It prefers names ending in `-development` (or the +configured `LOCALAI_BACKEND_DEV_SUFFIX`). Within that group, existing hardware +ranking and lexical-name ordering still apply. If no eligible development build +exists, LocalAI selects an eligible non-development build. If neither group has +an eligible build, it does not register the alias. + +For example, generic `example-engine` selects `cpu-example-engine-development` +over `cpu-example-engine` when both are installed and compatible. A compatible +CPU development build also takes priority over a stable CUDA build. +An incompatible development build cannot take priority over a compatible stable +build. Compatibility uses LocalAI's existing hardware heuristics, including the +`LOCALAI_FORCE_META_BACKEND_CAPABILITY=disable` override; it does not guarantee +that native libraries can load. + +Explicit concrete names keep their original paths. Selecting +`cpu-example-engine` does not redirect to its development counterpart. +Installing a development image under a stable name does not rename that entry. +The preference also retains its existing development-first download behavior. + +The **Settings** value `prefer_development_backends` supplies the frontend's +preference at its next startup. The API reports a restart requirement when you save this value. Neither API +updates nor edits to `runtime_settings.json` change +registered paths or restart running models. The displayed preference can change +before the resolver adopts it. Startup uses the existing settings merge: +explicit default-valued CLI or environment values, including `false`, can be +indistinguishable from unset and do not necessarily override a saved `true`. + +For distributed `local-ai worker`, set the flag or environment variable on each +worker, then restart it. Frontend flags and Settings changes do not configure +remote workers. Legacy specialized worker subcommands do not expose this +worker-local option. + ## Managing Backends in the UI The **Operate → Runtime → Backends** page is the canonical home for the diff --git a/pkg/system/state.go b/pkg/system/state.go index e631c5c33..7139560f2 100644 --- a/pkg/system/state.go +++ b/pkg/system/state.go @@ -29,9 +29,11 @@ type SystemState struct { BackendImagesReleaseTag string BackendImagesBranchTag string BackendDevSuffix string - // PreferDevelopmentBackends installs the development image as the primary + // PreferDevelopmentBackends prefers compatible runnable installed development + // builds for generic aliases and installs the development image as the primary // backend URI (the released image becomes a fallback) rather than only using // development as a download fallback when the released image is missing. + // This resolver preference is fixed at startup, not hot-switched by settings. PreferDevelopmentBackends bool // RequireBackendIntegrity is the strict-integrity switch bound to