From 6cfc99196d0e7ac09346267b887ae8da83eaeb3d Mon Sep 17 00:00:00 2001 From: "Plamen K. Kosseff" <333840+blackd@users.noreply.github.com> Date: Sat, 26 Sep 2026 14:33:51 +0300 Subject: [PATCH] feat(gallery): read metadata for system-path backends, enabling variant aliases (#12141) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(gallery): read metadata for system-path backends, enabling variant aliases Problem: - ListSystemBackends only read metadata.json for user-managed backends; the system-path scan (LOCALAI_BACKENDS_SYSTEM_PATH) was a bare directory walk with Metadata hardcoded nil - system-packaged backends (distro packages installing several accelerator builds of one backend) could not declare aliases or meta indirection at all, while gallery-installed backends could - surfaced while packaging LocalAI for Gentoo: the packages install cpu-/rocm-/vulkan-audio-cpp as system backends aliased to audio-cpp, which the server ignored Change: - scan each root separately, clean the system collection against the user-managed one, merge, then build and resolve — precedence lives in one explicit step - alias candidates carry their own metadata: the resolved alias entry can never pair one installation's executable with another's metadata, and it reports the chosen candidate's origin (IsSystem) - deterministic resolution: entries build in sorted name order and candidates sort by name at the resolution site, independent of scan order Precedence (user-managed always wins): - a user-managed backend hides a same-named system backend entirely - a user-managed variant takes over its whole alias family: the alias resolves among user-managed variants only and the system family's concrete names disappear — family versions move together, and a stale system variant may not work with newer models, so it must not stay reachable - a system variant's alias never hijacks a name that exists as a user-managed backend Tests: Ginkgo regressions for system-path aliasing, same-name hiding, family takeover, and the full metadata permutation matrix of cross-root name collisions (both directions, with and without metadata on each side). Docs: new "Backend Directory Format" section (run.sh, metadata.json, alias resolution — previously undocumented for user-managed backends too) and "System-Provided Backends" with the precedence rules. Assisted-by: Claude:claude-fable-5 Signed-off-by: Plamen K. Kosseff * fix(gallery): preserve managed meta backends A system alias can replace a user-managed meta backend during discovery. Protect meta entries with the same precedence guard as concrete backends. Add a regression test and clarify the documented precedence. Assisted-by: Codex:GPT-6 Signed-off-by: Plamen K. Kosseff --------- Signed-off-by: Plamen K. Kosseff Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/gallery/backends.go | 183 ++++++++++----- core/gallery/backends_test.go | 378 ++++++++++++++++++++++++++++++ docs/content/features/backends.md | 69 ++++++ 3 files changed, 569 insertions(+), 61 deletions(-) diff --git a/core/gallery/backends.go b/core/gallery/backends.go index bcb021ed0..21159eb21 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "time" @@ -86,8 +87,10 @@ func developmentURI(uri, latestTag, masterTag string) (string, bool) { // backendCandidate represents an installed concrete backend option for a given alias type backendCandidate struct { - name string - runFile string + name string + runFile string + isSystem bool + metadata *BackendMetadata } // readBackendMetadata reads the metadata JSON file for a backend @@ -635,65 +638,110 @@ func (b SystemBackends) GetAll() []SystemBackend { return backends } +// collectedBackend is one backend directory as scanned from its root, +// before cross-root cleaning. +type collectedBackend struct { + basePath string + metadata *BackendMetadata + isSystem bool +} + +// collectBackendDir reads one backend directory's optional metadata.json +// into the root's collection. +func collectBackendDir(basePath, dir string, isSystem bool, entries map[string]collectedBackend) error { + metadata := &BackendMetadata{Name: dir} + if _, err := os.Stat(filepath.Join(basePath, dir, metadataFile)); err == nil { + m, rerr := readBackendMetadata(filepath.Join(basePath, dir)) + if rerr != nil { + return rerr + } + if m != nil { + metadata = m + } + } + entries[dir] = collectedBackend{basePath: basePath, metadata: metadata, isSystem: isSystem} + return nil +} + +// collectRoot scans one backends root into its own collection. Metadata +// errors: warn-and-skip when lenient (system root), hard error otherwise +// (user-managed root). +func collectRoot(basePath string, isSystem, lenient bool) (map[string]collectedBackend, error) { + entries := make(map[string]collectedBackend) + dirEntries, err := os.ReadDir(basePath) + if err != nil { + return entries, err + } + for _, e := range dirEntries { + if !e.IsDir() { + continue + } + if cerr := collectBackendDir(basePath, e.Name(), isSystem, entries); cerr != nil { + if !lenient { + return nil, cerr + } + xlog.Warn("Skipping backend with unreadable metadata", "dir", e.Name(), "error", cerr) + } + } + return entries, nil +} + func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) { - // Gather backends from system and user paths, then resolve alias conflicts by capability. backends := make(SystemBackends) - // System-provided backends - if systemBackends, err := os.ReadDir(systemState.Backend.BackendsSystemPath); err == nil { - for _, systemBackend := range systemBackends { - if systemBackend.IsDir() { - run := filepath.Join(systemState.Backend.BackendsSystemPath, systemBackend.Name(), runFile) - if _, err := os.Stat(run); err == nil { - backends[systemBackend.Name()] = SystemBackend{ - Name: systemBackend.Name(), - RunFile: run, - IsMeta: false, - IsSystem: true, - Metadata: nil, - } - } - } + // 1. Scan each root separately. + systemEntries, err := collectRoot(systemState.Backend.BackendsSystemPath, true, true) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + xlog.Debug("No system backends found") + } else { + xlog.Warn("Failed to read system backends, proceeding with user-managed backends", "error", err) } - } else if !errors.Is(err, os.ErrNotExist) { - xlog.Warn("Failed to read system backends, proceeding with user-managed backends", "error", err) - } else if errors.Is(err, os.ErrNotExist) { - xlog.Debug("No system backends found") } - - // User-managed backends and alias collection - entries, err := os.ReadDir(systemState.Backend.BackendsPath) + managedEntries, err := collectRoot(systemState.Backend.BackendsPath, false, false) if err != nil { return nil, err } + // 2. Clean the system collection against the user-managed one: + // - a user-managed backend shadows a same-named system backend + // - a user-managed alias member takes over its whole alias group: + // resolution must never mix the roots (and their possibly + // different versions) within one backend family, and a stale + // system variant may not work with newer models, so the whole + // system family becomes invisible, concrete names included + managedAliases := make(map[string]bool) + for _, e := range managedEntries { + if e.metadata.Alias != "" { + managedAliases[e.metadata.Alias] = true + } + } + for name, e := range systemEntries { + if _, shadowed := managedEntries[name]; shadowed { + delete(systemEntries, name) + } else if e.metadata.Alias != "" && managedAliases[e.metadata.Alias] { + delete(systemEntries, name) + } + } + + // 3. Merge — disjoint by construction after cleaning. + entriesByDir := managedEntries + for name, e := range systemEntries { + entriesByDir[name] = e + } + + // 4. Build concrete entries, alias candidacies and meta indirection + // from the merged collection, in sorted order. + dirs := make([]string, 0, len(entriesByDir)) + for dir := range entriesByDir { + dirs = append(dirs, dir) + } + slices.Sort(dirs) + aliasGroups := make(map[string][]backendCandidate) - metaMap := make(map[string]*BackendMetadata) - - for _, e := range entries { - if !e.IsDir() { - continue - } - dir := e.Name() - run := filepath.Join(systemState.Backend.BackendsPath, dir, runFile) - - var metadata *BackendMetadata - metadataPath := filepath.Join(systemState.Backend.BackendsPath, dir, metadataFile) - if _, err := os.Stat(metadataPath); os.IsNotExist(err) { - metadata = &BackendMetadata{Name: dir} - } else { - m, rerr := readBackendMetadata(filepath.Join(systemState.Backend.BackendsPath, dir)) - if rerr != nil { - return nil, rerr - } - if m == nil { - metadata = &BackendMetadata{Name: dir} - } else { - metadata = m - } - } - - metaMap[dir] = metadata + for _, dir := range dirs { + entry := entriesByDir[dir] + run := filepath.Join(entry.basePath, dir, runFile) // Concrete-backend entry if _, err := os.Stat(run); err == nil { @@ -701,22 +749,25 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) Name: dir, RunFile: run, IsMeta: false, - Metadata: metadata, + IsSystem: entry.isSystem, + Metadata: entry.metadata, } } // Alias candidates - if metadata.Alias != "" { - aliasGroups[metadata.Alias] = append(aliasGroups[metadata.Alias], backendCandidate{name: dir, runFile: run}) + if entry.metadata.Alias != "" { + aliasGroups[entry.metadata.Alias] = append(aliasGroups[entry.metadata.Alias], + backendCandidate{name: dir, runFile: run, isSystem: entry.isSystem, metadata: entry.metadata}) } // Meta backends indirection - if metadata.MetaBackendFor != "" { - backends[metadata.Name] = SystemBackend{ - Name: metadata.Name, - RunFile: filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor, runFile), + if entry.metadata.MetaBackendFor != "" { + backends[entry.metadata.Name] = SystemBackend{ + Name: entry.metadata.Name, + RunFile: filepath.Join(entry.basePath, entry.metadata.MetaBackendFor, runFile), IsMeta: true, - Metadata: metadata, + IsSystem: entry.isSystem, + Metadata: entry.metadata, } } } @@ -724,6 +775,11 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) // Resolve aliases using system capability preferences tokens := systemState.BackendPreferenceTokens() for alias, cands := range aliasGroups { + // First-token-match depends on candidate order: sort by name so + // resolution is deterministic by construction, not by scan order. + slices.SortFunc(cands, func(a, b backendCandidate) int { + return strings.Compare(a.name, b.name) + }) chosen := backendCandidate{} // Try preference tokens for _, t := range tokens { @@ -749,12 +805,17 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) if chosen.runFile == "" { continue } - md := metaMap[chosen.name] + if existing, ok := backends[alias]; ok && !existing.IsSystem && chosen.isSystem { + // A system-derived alias never hijacks a user-managed + // backend of the same name, including meta indirection. + continue + } backends[alias] = SystemBackend{ Name: alias, RunFile: chosen.runFile, IsMeta: false, - Metadata: md, + IsSystem: chosen.isSystem, + Metadata: chosen.metadata, } } diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index 4424c165c..ec455d582 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -111,6 +111,384 @@ var _ = Describe("Runtime capability-based backend selection", func() { })) }) + It("ListSystemBackends resolves aliases for system-path backends", func() { + must := func(err error) { Expect(err).NotTo(HaveOccurred()) } + + sysRoot, err := os.MkdirTemp("", "system-backends-*") + must(err) + defer os.RemoveAll(sysRoot) + managedRoot, err := os.MkdirTemp("", "managed-backends-*") + must(err) + defer os.RemoveAll(managedRoot) + + for _, name := range []string{"cpu-audio-cpp", "cuda12-audio-cpp"} { + dir := filepath.Join(sysRoot, name) + must(os.MkdirAll(dir, 0o750)) + b, _ := json.Marshal(&BackendMetadata{Alias: "audio-cpp", Name: name}) + must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644)) + must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755)) + } + + must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia")) + defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }() + + state, err := system.GetSystemState( + system.WithBackendPath(managedRoot), + system.WithBackendSystemPath(sysRoot), + ) + must(err) + state.GPUVendor = "nvidia" + + backs, err := ListSystemBackends(state) + must(err) + + aliasBack, ok := backs.Get("audio-cpp") + Expect(ok).To(BeTrue()) + Expect(aliasBack.RunFile).To(Equal(filepath.Join(sysRoot, "cuda12-audio-cpp", "run.sh"))) + Expect(aliasBack.IsSystem).To(BeTrue()) + + concrete, ok := backs.Get("cpu-audio-cpp") + Expect(ok).To(BeTrue()) + Expect(concrete.IsSystem).To(BeTrue()) + Expect(concrete.Metadata).NotTo(BeNil()) + Expect(concrete.Metadata.Alias).To(Equal("audio-cpp")) + }) + + It("ListSystemBackends lets a user-managed backend hide a same-named system backend", func() { + must := func(err error) { Expect(err).NotTo(HaveOccurred()) } + + mkBackend := func(root, name, alias string) string { + dir := filepath.Join(root, name) + must(os.MkdirAll(dir, 0o750)) + b, _ := json.Marshal(&BackendMetadata{Alias: alias, Name: name}) + must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644)) + must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755)) + return dir + } + + sysRoot, err := os.MkdirTemp("", "system-backends-*") + must(err) + defer os.RemoveAll(sysRoot) + managedRoot, err := os.MkdirTemp("", "managed-backends-*") + must(err) + defer os.RemoveAll(managedRoot) + + // The SAME concrete name exists in both roots. + mkBackend(sysRoot, "cpu-audio-cpp", "audio-cpp") + managedDir := mkBackend(managedRoot, "cpu-audio-cpp", "audio-cpp") + + must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "cpu")) + defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }() + + state, err := system.GetSystemState( + system.WithBackendPath(managedRoot), + system.WithBackendSystemPath(sysRoot), + ) + must(err) + + backs, err := ListSystemBackends(state) + must(err) + + // Concrete name and alias BOTH run the managed installation and + // report its metadata — never the system executable with managed + // metadata (or any other cross-root mix). + concrete, ok := backs.Get("cpu-audio-cpp") + Expect(ok).To(BeTrue()) + Expect(concrete.RunFile).To(Equal(filepath.Join(managedDir, "run.sh"))) + Expect(concrete.IsSystem).To(BeFalse()) + + aliasBack, ok := backs.Get("audio-cpp") + Expect(ok).To(BeTrue()) + Expect(aliasBack.RunFile).To(Equal(filepath.Join(managedDir, "run.sh"))) + Expect(aliasBack.IsSystem).To(BeFalse()) + Expect(aliasBack.Metadata).NotTo(BeNil()) + Expect(aliasBack.Metadata.Name).To(Equal("cpu-audio-cpp")) + }) + + It("ListSystemBackends preserves managed meta backends against system aliases", func() { + sysRoot := filepath.Join(tempDir, "system") + managedRoot := filepath.Join(tempDir, "managed") + for _, dir := range []string{ + filepath.Join(sysRoot, "cpu-audio-cpp"), + filepath.Join(managedRoot, "custom-audio"), + filepath.Join(managedRoot, "audio-cpp"), + } { + Expect(os.MkdirAll(dir, 0o750)).To(Succeed()) + } + Expect(os.WriteFile(filepath.Join(sysRoot, "cpu-audio-cpp", "run.sh"), nil, 0o755)).To(Succeed()) + managedRun := filepath.Join(managedRoot, "custom-audio", "run.sh") + Expect(os.WriteFile(managedRun, nil, 0o755)).To(Succeed()) + Expect(writeBackendMetadata(filepath.Join(sysRoot, "cpu-audio-cpp"), &BackendMetadata{ + Name: "cpu-audio-cpp", Alias: "audio-cpp", + })).To(Succeed()) + Expect(writeBackendMetadata(filepath.Join(managedRoot, "audio-cpp"), &BackendMetadata{ + Name: "audio-cpp", MetaBackendFor: "custom-audio", + })).To(Succeed()) + state, err := system.GetSystemState( + system.WithBackendPath(managedRoot), + system.WithBackendSystemPath(sysRoot), + ) + Expect(err).NotTo(HaveOccurred()) + + backends, err := ListSystemBackends(state) + Expect(err).NotTo(HaveOccurred()) + backend, ok := backends.Get("audio-cpp") + Expect(ok).To(BeTrue()) + Expect(backend.RunFile).To(Equal(managedRun)) + Expect(backend.IsSystem).To(BeFalse()) + Expect(backend.IsMeta).To(BeTrue()) + Expect(backend.Metadata.MetaBackendFor).To(Equal("custom-audio")) + }) + + It("ListSystemBackends lets a user-managed variant take over its whole alias group", func() { + must := func(err error) { Expect(err).NotTo(HaveOccurred()) } + + mkBackend := func(root, name, alias string) string { + dir := filepath.Join(root, name) + must(os.MkdirAll(dir, 0o750)) + b, _ := json.Marshal(&BackendMetadata{Alias: alias, Name: name}) + must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644)) + must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755)) + return dir + } + + sysRoot, err := os.MkdirTemp("", "system-backends-*") + must(err) + defer os.RemoveAll(sysRoot) + managedRoot, err := os.MkdirTemp("", "managed-backends-*") + must(err) + defer os.RemoveAll(managedRoot) + + // System family with the variant an NVIDIA host would prefer; + // the user installs only the cpu variant via the gallery. + mkBackend(sysRoot, "cuda12-audio-cpp", "audio-cpp") + mkBackend(sysRoot, "vulkan-audio-cpp", "audio-cpp") + managedDir := mkBackend(managedRoot, "cpu-audio-cpp", "audio-cpp") + + must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia")) + defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }() + + state, err := system.GetSystemState( + system.WithBackendPath(managedRoot), + system.WithBackendSystemPath(sysRoot), + ) + must(err) + state.GPUVendor = "nvidia" + + backs, err := ListSystemBackends(state) + must(err) + + // The alias resolves to the managed variant even though the + // capability tokens would prefer the system cuda build: a + // gallery-installed variant takes over its whole family. + aliasBack, ok := backs.Get("audio-cpp") + Expect(ok).To(BeTrue()) + Expect(aliasBack.RunFile).To(Equal(filepath.Join(managedDir, "run.sh"))) + Expect(aliasBack.IsSystem).To(BeFalse()) + + // The system family is invisible entirely, concrete names + // included — a stale system variant may not work with newer + // models, so it must not stay reachable. + _, ok = backs.Get("cuda12-audio-cpp") + Expect(ok).To(BeFalse()) + _, ok = backs.Get("vulkan-audio-cpp") + Expect(ok).To(BeFalse()) + }) + + It("ListSystemBackends resolves a mixed population across both roots", func() { + must := func(err error) { Expect(err).NotTo(HaveOccurred()) } + + // alias == "" installs no metadata.json at all (old-style dir). + mkBackend := func(root, name, alias string) string { + dir := filepath.Join(root, name) + must(os.MkdirAll(dir, 0o750)) + if alias != "" { + b, _ := json.Marshal(&BackendMetadata{Alias: alias, Name: name}) + must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644)) + } + must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755)) + return dir + } + + sysRoot, err := os.MkdirTemp("", "system-backends-*") + must(err) + defer os.RemoveAll(sysRoot) + managedRoot, err := os.MkdirTemp("", "managed-backends-*") + must(err) + defer os.RemoveAll(managedRoot) + + // System root: two aliased families and a metadata-less loner. + mkBackend(sysRoot, "cpu-audio-cpp", "audio-cpp") + mkBackend(sysRoot, "cuda12-audio-cpp", "audio-cpp") + mkBackend(sysRoot, "cpu-llama-cpp", "llama-cpp") + sysVulkanLlama := mkBackend(sysRoot, "vulkan-llama-cpp", "llama-cpp") + mkBackend(sysRoot, "piper", "") + + // Managed root: a family takeover, a metadata-less name + // collision, a metadata-less variant that must NOT take over + // its would-be family, a plain loner, and its own family. + managedCPUAudio := mkBackend(managedRoot, "cpu-audio-cpp", "audio-cpp") + managedWhisper := mkBackend(managedRoot, "whisper", "") + mkBackend(sysRoot, "whisper", "") + mkBackend(managedRoot, "rocm-llama-cpp", "") + mkBackend(managedRoot, "bark", "") + managedKokoro := mkBackend(managedRoot, "kokoro", "tts-suite") + + must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia")) + defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }() + + state, err := system.GetSystemState( + system.WithBackendPath(managedRoot), + system.WithBackendSystemPath(sysRoot), + ) + must(err) + state.GPUVendor = "nvidia" + + backs, err := ListSystemBackends(state) + must(err) + + expectBackend := func(name, runFile string, isSystem bool) { + b, ok := backs.Get(name) + Expect(ok).To(BeTrue(), name) + Expect(b.RunFile).To(Equal(runFile), name) + Expect(b.IsSystem).To(Equal(isSystem), name) + } + expectGone := func(name string) { + _, ok := backs.Get(name) + Expect(ok).To(BeFalse(), name) + } + + // audio-cpp family: the managed cpu variant takes over — the + // alias resolves to it despite the nvidia tokens preferring + // cuda, and the system family is gone, concretes included. + expectBackend("audio-cpp", filepath.Join(managedCPUAudio, "run.sh"), false) + expectBackend("cpu-audio-cpp", filepath.Join(managedCPUAudio, "run.sh"), false) + expectGone("cuda12-audio-cpp") + + // llama-cpp family: the managed rocm-llama-cpp declares NO + // metadata, so it joins no family and triggers no takeover — + // the system family stays and resolves by capability (vulkan + // on this nvidia host, no cuda variant present). + expectBackend("llama-cpp", filepath.Join(sysVulkanLlama, "run.sh"), true) + expectBackend("cpu-llama-cpp", filepath.Join(sysRoot, "cpu-llama-cpp", "run.sh"), true) + expectBackend("vulkan-llama-cpp", filepath.Join(sysVulkanLlama, "run.sh"), true) + expectBackend("rocm-llama-cpp", filepath.Join(managedRoot, "rocm-llama-cpp", "run.sh"), false) + + // whisper: same metadata-less name in both roots — managed + // hides system. + expectBackend("whisper", filepath.Join(managedWhisper, "run.sh"), false) + + // Loners survive untouched on their own side. + expectBackend("piper", filepath.Join(sysRoot, "piper", "run.sh"), true) + expectBackend("bark", filepath.Join(managedRoot, "bark", "run.sh"), false) + + // A managed-only alias group resolves within itself. + expectBackend("tts-suite", filepath.Join(managedKokoro, "run.sh"), false) + expectBackend("kokoro", filepath.Join(managedKokoro, "run.sh"), false) + }) + + It("ListSystemBackends: name collisions across roots, all metadata permutations", func() { + must := func(err error) { Expect(err).NotTo(HaveOccurred()) } + + // withMeta installs metadata.json; alias may be empty (name-only + // metadata — possible and legal, e.g. a packaged backend that + // declares no family). + mk := func(root, name, alias string, withMeta bool) string { + dir := filepath.Join(root, name) + must(os.MkdirAll(dir, 0o750)) + if withMeta { + b, _ := json.Marshal(&BackendMetadata{Alias: alias, Name: name}) + must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644)) + } + must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755)) + return dir + } + + sysRoot, err := os.MkdirTemp("", "system-backends-*") + must(err) + defer os.RemoveAll(sysRoot) + managedRoot, err := os.MkdirTemp("", "managed-backends-*") + must(err) + defer os.RemoveAll(managedRoot) + + // 1a. Metadata on BOTH sides, aliased side managed: + // system concrete alpha (name-only metadata) vs managed + // cuda-alpha aliased to alpha. + mk(sysRoot, "alpha", "", true) + cudaAlpha := mk(managedRoot, "cuda-alpha", "alpha", true) + + // 1b. Reverse: managed concrete beta (name-only metadata) vs + // system cuda-beta aliased to beta. + beta := mk(managedRoot, "beta", "", true) + mk(sysRoot, "cuda-beta", "beta", true) + + // 2a. No metadata on ONE side (system): bare system gamma vs + // managed cuda-gamma aliased to gamma. + mk(sysRoot, "gamma", "", false) + cudaGamma := mk(managedRoot, "cuda-gamma", "gamma", true) + + // 2b. Reverse: bare managed delta vs system cuda-delta aliased + // to delta. + delta := mk(managedRoot, "delta", "", false) + mk(sysRoot, "cuda-delta", "delta", true) + + // 3a. Same name in BOTH roots, metadata on both. + epsilonManaged := mk(managedRoot, "epsilon", "", true) + mk(sysRoot, "epsilon", "", true) + + // 3b. Same name in BOTH roots, metadata on neither. + zetaManaged := mk(managedRoot, "zeta", "", false) + mk(sysRoot, "zeta", "", false) + + must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia")) + defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }() + + state, err := system.GetSystemState( + system.WithBackendPath(managedRoot), + system.WithBackendSystemPath(sysRoot), + ) + must(err) + state.GPUVendor = "nvidia" + + backs, err := ListSystemBackends(state) + must(err) + + expect := func(name, runFile string, isSystem bool) { + b, ok := backs.Get(name) + Expect(ok).To(BeTrue(), name) + Expect(b.RunFile).To(Equal(filepath.Join(runFile, "run.sh")), name) + Expect(b.IsSystem).To(Equal(isSystem), name) + } + + // 1a: the managed alias owns the name; the system concrete + // (metadata or not) is unreachable. + expect("alpha", cudaAlpha, false) + expect("cuda-alpha", cudaAlpha, false) + + // 1b: the managed concrete keeps its name — a system alias + // never hijacks it; the system variant stays concrete-only. + expect("beta", beta, false) + b, ok := backs.Get("cuda-beta") + Expect(ok).To(BeTrue()) + Expect(b.IsSystem).To(BeTrue()) + + // 2a: same as 1a — the system concrete's missing metadata + // changes nothing. + expect("gamma", cudaGamma, false) + + // 2b: same as 1b — the managed concrete's missing metadata + // changes nothing. + expect("delta", delta, false) + b, ok = backs.Get("cuda-delta") + Expect(ok).To(BeTrue()) + Expect(b.IsSystem).To(BeTrue()) + + // 3a/3b: plain same-name hiding, managed wins, with or + // without metadata on either side. + expect("epsilon", epsilonManaged, false) + expect("zeta", zetaManaged, false) + }) + It("ListSystemBackends prefers optimal alias candidate", func() { // Arrange two installed backends sharing the same alias must := func(err error) { Expect(err).NotTo(HaveOccurred()) } diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 69ae0cf0f..6085cf536 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -148,6 +148,75 @@ export LOCALAI_EXTERNAL_BACKENDS="llm-backend,diffusion-backend" local-ai run ``` +## Backend Directory Format + +Every backend, whether the gallery installed it into the user-managed +location or a system package shipped it, is a directory with one +required file: + +- `run.sh` — the entry point LocalAI executes to start the backend. + +and one optional file, `metadata.json`: + +```json +{ + "name": "rocm-audio-cpp", + "alias": "audio-cpp" +} +``` + +- `name` — the concrete backend name (defaults to the directory name). +- `alias` — registers this directory as a *variant* of a backend + family. When several installed variants share an alias + (`cpu-audio-cpp`, `rocm-audio-cpp`, ... all aliased to `audio-cpp`), + a model config using `backend: audio-cpp` resolves to the variant + best matching the host's capability (CUDA before Vulkan before CPU + on an NVIDIA host, ROCm first on AMD, and so on). Each variant also + stays individually addressable by its concrete name, e.g. + `backend: cpu-audio-cpp` to keep VRAM free for other models. +- `meta_backend_for` — points a meta entry at a concrete backend + directory installed next to it. + +A directory without `metadata.json` is a plain backend under its +directory name. Gallery installs write this metadata automatically +(with additional bookkeeping fields such as `gallery_url` and +`installed_at`); it only needs writing by hand when packaging backends +outside the gallery. + +## System-Provided Backends + +Backends do not have to come from the gallery: directories under +`LOCALAI_BACKENDS_SYSTEM_PATH` (default `/var/lib/local-ai/backends`) +are discovered on every scan, using the same +[directory format](#backend-directory-format) as user-managed +backends. This is the integration point for distribution packages — +the package manager installs backends there, while gallery installs +keep living in the user-managed `LOCALAI_BACKENDS_PATH`. + +One difference in error handling: a system directory with unreadable +metadata is skipped with a warning, while unreadable metadata in the +user-managed location fails the listing — a system package must never +be able to break the discovery of the user's own backends. + +### Precedence between the two locations + +User-managed backends always win over system-provided ones: + +- **Same name in both locations** — the user-managed backend hides the + system one entirely. +- **Family takeover** — installing *any* variant of an alias family + into the user-managed location (e.g. from the gallery) replaces the + whole system family: the alias resolves only among user-managed + variants, and the system family's concrete names disappear from the + listing. Variants of one family are versioned together; resolution + never mixes installations of different origins within a family, and + a stale system variant is not kept reachable. +- **Names never get hijacked** — a system variant's alias cannot take + over a name that exists as a user-managed backend (including a meta + backend): `backend: + audio-cpp` keeps running the user's `audio-cpp` installation even if + a system package later ships variants aliased to that name. + ## Creating a Backend To create a new backend, you need to: