mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-27 08:35:11 -04:00
feat(gallery): read metadata for system-path backends, enabling variant aliases (#12141)
* feat(gallery): read metadata for system-path backends, enabling variant aliases Problem: - ListSystemBackends only read metadata.json for user-managed backends; the system-path scan (LOCALAI_BACKENDS_SYSTEM_PATH) was a bare directory walk with Metadata hardcoded nil - system-packaged backends (distro packages installing several accelerator builds of one backend) could not declare aliases or meta indirection at all, while gallery-installed backends could - surfaced while packaging LocalAI for Gentoo: the packages install cpu-/rocm-/vulkan-audio-cpp as system backends aliased to audio-cpp, which the server ignored Change: - scan each root separately, clean the system collection against the user-managed one, merge, then build and resolve — precedence lives in one explicit step - alias candidates carry their own metadata: the resolved alias entry can never pair one installation's executable with another's metadata, and it reports the chosen candidate's origin (IsSystem) - deterministic resolution: entries build in sorted name order and candidates sort by name at the resolution site, independent of scan order Precedence (user-managed always wins): - a user-managed backend hides a same-named system backend entirely - a user-managed variant takes over its whole alias family: the alias resolves among user-managed variants only and the system family's concrete names disappear — family versions move together, and a stale system variant may not work with newer models, so it must not stay reachable - a system variant's alias never hijacks a name that exists as a user-managed backend Tests: Ginkgo regressions for system-path aliasing, same-name hiding, family takeover, and the full metadata permutation matrix of cross-root name collisions (both directions, with and without metadata on each side). Docs: new "Backend Directory Format" section (run.sh, metadata.json, alias resolution — previously undocumented for user-managed backends too) and "System-Provided Backends" with the precedence rules. Assisted-by: Claude:claude-fable-5 Signed-off-by: Plamen K. Kosseff <p.kosseff@gmail.com> * fix(gallery): preserve managed meta backends A system alias can replace a user-managed meta backend during discovery. Protect meta entries with the same precedence guard as concrete backends. Add a regression test and clarify the documented precedence. Assisted-by: Codex:GPT-6 Signed-off-by: Plamen K. Kosseff <p.kosseff@gmail.com> --------- Signed-off-by: Plamen K. Kosseff <p.kosseff@gmail.com> Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
This commit is contained in:
Plamen K. Kosseffandlocalai-org-maint-bot authored and GitHub committed
1 parent
9fa672faee
commit
6cfc99196d
3 files changed
+569
-61
No files matched your search
+122
-61
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -86,8 +87,10 @@ func developmentURI(uri, latestTag, masterTag string) (string, bool) {
|
||||
|
||||
// backendCandidate represents an installed concrete backend option for a given alias
|
||||
type backendCandidate struct {
|
||||
name string
|
||||
runFile string
|
||||
name string
|
||||
runFile string
|
||||
isSystem bool
|
||||
metadata *BackendMetadata
|
||||
}
|
||||
|
||||
// readBackendMetadata reads the metadata JSON file for a backend
|
||||
@@ -635,65 +638,110 @@ func (b SystemBackends) GetAll() []SystemBackend {
|
||||
return backends
|
||||
}
|
||||
|
||||
// collectedBackend is one backend directory as scanned from its root,
|
||||
// before cross-root cleaning.
|
||||
type collectedBackend struct {
|
||||
basePath string
|
||||
metadata *BackendMetadata
|
||||
isSystem bool
|
||||
}
|
||||
|
||||
// collectBackendDir reads one backend directory's optional metadata.json
|
||||
// into the root's collection.
|
||||
func collectBackendDir(basePath, dir string, isSystem bool, entries map[string]collectedBackend) error {
|
||||
metadata := &BackendMetadata{Name: dir}
|
||||
if _, err := os.Stat(filepath.Join(basePath, dir, metadataFile)); err == nil {
|
||||
m, rerr := readBackendMetadata(filepath.Join(basePath, dir))
|
||||
if rerr != nil {
|
||||
return rerr
|
||||
}
|
||||
if m != nil {
|
||||
metadata = m
|
||||
}
|
||||
}
|
||||
entries[dir] = collectedBackend{basePath: basePath, metadata: metadata, isSystem: isSystem}
|
||||
return nil
|
||||
}
|
||||
|
||||
// collectRoot scans one backends root into its own collection. Metadata
|
||||
// errors: warn-and-skip when lenient (system root), hard error otherwise
|
||||
// (user-managed root).
|
||||
func collectRoot(basePath string, isSystem, lenient bool) (map[string]collectedBackend, error) {
|
||||
entries := make(map[string]collectedBackend)
|
||||
dirEntries, err := os.ReadDir(basePath)
|
||||
if err != nil {
|
||||
return entries, err
|
||||
}
|
||||
for _, e := range dirEntries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
if cerr := collectBackendDir(basePath, e.Name(), isSystem, entries); cerr != nil {
|
||||
if !lenient {
|
||||
return nil, cerr
|
||||
}
|
||||
xlog.Warn("Skipping backend with unreadable metadata", "dir", e.Name(), "error", cerr)
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) {
|
||||
// Gather backends from system and user paths, then resolve alias conflicts by capability.
|
||||
backends := make(SystemBackends)
|
||||
|
||||
// System-provided backends
|
||||
if systemBackends, err := os.ReadDir(systemState.Backend.BackendsSystemPath); err == nil {
|
||||
for _, systemBackend := range systemBackends {
|
||||
if systemBackend.IsDir() {
|
||||
run := filepath.Join(systemState.Backend.BackendsSystemPath, systemBackend.Name(), runFile)
|
||||
if _, err := os.Stat(run); err == nil {
|
||||
backends[systemBackend.Name()] = SystemBackend{
|
||||
Name: systemBackend.Name(),
|
||||
RunFile: run,
|
||||
IsMeta: false,
|
||||
IsSystem: true,
|
||||
Metadata: nil,
|
||||
}
|
||||
}
|
||||
}
|
||||
// 1. Scan each root separately.
|
||||
systemEntries, err := collectRoot(systemState.Backend.BackendsSystemPath, true, true)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
xlog.Debug("No system backends found")
|
||||
} else {
|
||||
xlog.Warn("Failed to read system backends, proceeding with user-managed backends", "error", err)
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
xlog.Warn("Failed to read system backends, proceeding with user-managed backends", "error", err)
|
||||
} else if errors.Is(err, os.ErrNotExist) {
|
||||
xlog.Debug("No system backends found")
|
||||
}
|
||||
|
||||
// User-managed backends and alias collection
|
||||
entries, err := os.ReadDir(systemState.Backend.BackendsPath)
|
||||
managedEntries, err := collectRoot(systemState.Backend.BackendsPath, false, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 2. Clean the system collection against the user-managed one:
|
||||
// - a user-managed backend shadows a same-named system backend
|
||||
// - a user-managed alias member takes over its whole alias group:
|
||||
// resolution must never mix the roots (and their possibly
|
||||
// different versions) within one backend family, and a stale
|
||||
// system variant may not work with newer models, so the whole
|
||||
// system family becomes invisible, concrete names included
|
||||
managedAliases := make(map[string]bool)
|
||||
for _, e := range managedEntries {
|
||||
if e.metadata.Alias != "" {
|
||||
managedAliases[e.metadata.Alias] = true
|
||||
}
|
||||
}
|
||||
for name, e := range systemEntries {
|
||||
if _, shadowed := managedEntries[name]; shadowed {
|
||||
delete(systemEntries, name)
|
||||
} else if e.metadata.Alias != "" && managedAliases[e.metadata.Alias] {
|
||||
delete(systemEntries, name)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Merge — disjoint by construction after cleaning.
|
||||
entriesByDir := managedEntries
|
||||
for name, e := range systemEntries {
|
||||
entriesByDir[name] = e
|
||||
}
|
||||
|
||||
// 4. Build concrete entries, alias candidacies and meta indirection
|
||||
// from the merged collection, in sorted order.
|
||||
dirs := make([]string, 0, len(entriesByDir))
|
||||
for dir := range entriesByDir {
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
slices.Sort(dirs)
|
||||
|
||||
aliasGroups := make(map[string][]backendCandidate)
|
||||
metaMap := make(map[string]*BackendMetadata)
|
||||
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
dir := e.Name()
|
||||
run := filepath.Join(systemState.Backend.BackendsPath, dir, runFile)
|
||||
|
||||
var metadata *BackendMetadata
|
||||
metadataPath := filepath.Join(systemState.Backend.BackendsPath, dir, metadataFile)
|
||||
if _, err := os.Stat(metadataPath); os.IsNotExist(err) {
|
||||
metadata = &BackendMetadata{Name: dir}
|
||||
} else {
|
||||
m, rerr := readBackendMetadata(filepath.Join(systemState.Backend.BackendsPath, dir))
|
||||
if rerr != nil {
|
||||
return nil, rerr
|
||||
}
|
||||
if m == nil {
|
||||
metadata = &BackendMetadata{Name: dir}
|
||||
} else {
|
||||
metadata = m
|
||||
}
|
||||
}
|
||||
|
||||
metaMap[dir] = metadata
|
||||
for _, dir := range dirs {
|
||||
entry := entriesByDir[dir]
|
||||
run := filepath.Join(entry.basePath, dir, runFile)
|
||||
|
||||
// Concrete-backend entry
|
||||
if _, err := os.Stat(run); err == nil {
|
||||
@@ -701,22 +749,25 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error)
|
||||
Name: dir,
|
||||
RunFile: run,
|
||||
IsMeta: false,
|
||||
Metadata: metadata,
|
||||
IsSystem: entry.isSystem,
|
||||
Metadata: entry.metadata,
|
||||
}
|
||||
}
|
||||
|
||||
// Alias candidates
|
||||
if metadata.Alias != "" {
|
||||
aliasGroups[metadata.Alias] = append(aliasGroups[metadata.Alias], backendCandidate{name: dir, runFile: run})
|
||||
if entry.metadata.Alias != "" {
|
||||
aliasGroups[entry.metadata.Alias] = append(aliasGroups[entry.metadata.Alias],
|
||||
backendCandidate{name: dir, runFile: run, isSystem: entry.isSystem, metadata: entry.metadata})
|
||||
}
|
||||
|
||||
// Meta backends indirection
|
||||
if metadata.MetaBackendFor != "" {
|
||||
backends[metadata.Name] = SystemBackend{
|
||||
Name: metadata.Name,
|
||||
RunFile: filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor, runFile),
|
||||
if entry.metadata.MetaBackendFor != "" {
|
||||
backends[entry.metadata.Name] = SystemBackend{
|
||||
Name: entry.metadata.Name,
|
||||
RunFile: filepath.Join(entry.basePath, entry.metadata.MetaBackendFor, runFile),
|
||||
IsMeta: true,
|
||||
Metadata: metadata,
|
||||
IsSystem: entry.isSystem,
|
||||
Metadata: entry.metadata,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -724,6 +775,11 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error)
|
||||
// Resolve aliases using system capability preferences
|
||||
tokens := systemState.BackendPreferenceTokens()
|
||||
for alias, cands := range aliasGroups {
|
||||
// First-token-match depends on candidate order: sort by name so
|
||||
// resolution is deterministic by construction, not by scan order.
|
||||
slices.SortFunc(cands, func(a, b backendCandidate) int {
|
||||
return strings.Compare(a.name, b.name)
|
||||
})
|
||||
chosen := backendCandidate{}
|
||||
// Try preference tokens
|
||||
for _, t := range tokens {
|
||||
@@ -749,12 +805,17 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error)
|
||||
if chosen.runFile == "" {
|
||||
continue
|
||||
}
|
||||
md := metaMap[chosen.name]
|
||||
if existing, ok := backends[alias]; ok && !existing.IsSystem && chosen.isSystem {
|
||||
// A system-derived alias never hijacks a user-managed
|
||||
// backend of the same name, including meta indirection.
|
||||
continue
|
||||
}
|
||||
backends[alias] = SystemBackend{
|
||||
Name: alias,
|
||||
RunFile: chosen.runFile,
|
||||
IsMeta: false,
|
||||
Metadata: md,
|
||||
IsSystem: chosen.isSystem,
|
||||
Metadata: chosen.metadata,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -111,6 +111,384 @@ var _ = Describe("Runtime capability-based backend selection", func() {
|
||||
}))
|
||||
})
|
||||
|
||||
It("ListSystemBackends resolves aliases for system-path backends", func() {
|
||||
must := func(err error) { Expect(err).NotTo(HaveOccurred()) }
|
||||
|
||||
sysRoot, err := os.MkdirTemp("", "system-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(sysRoot)
|
||||
managedRoot, err := os.MkdirTemp("", "managed-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(managedRoot)
|
||||
|
||||
for _, name := range []string{"cpu-audio-cpp", "cuda12-audio-cpp"} {
|
||||
dir := filepath.Join(sysRoot, name)
|
||||
must(os.MkdirAll(dir, 0o750))
|
||||
b, _ := json.Marshal(&BackendMetadata{Alias: "audio-cpp", Name: name})
|
||||
must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644))
|
||||
must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755))
|
||||
}
|
||||
|
||||
must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia"))
|
||||
defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }()
|
||||
|
||||
state, err := system.GetSystemState(
|
||||
system.WithBackendPath(managedRoot),
|
||||
system.WithBackendSystemPath(sysRoot),
|
||||
)
|
||||
must(err)
|
||||
state.GPUVendor = "nvidia"
|
||||
|
||||
backs, err := ListSystemBackends(state)
|
||||
must(err)
|
||||
|
||||
aliasBack, ok := backs.Get("audio-cpp")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(aliasBack.RunFile).To(Equal(filepath.Join(sysRoot, "cuda12-audio-cpp", "run.sh")))
|
||||
Expect(aliasBack.IsSystem).To(BeTrue())
|
||||
|
||||
concrete, ok := backs.Get("cpu-audio-cpp")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(concrete.IsSystem).To(BeTrue())
|
||||
Expect(concrete.Metadata).NotTo(BeNil())
|
||||
Expect(concrete.Metadata.Alias).To(Equal("audio-cpp"))
|
||||
})
|
||||
|
||||
It("ListSystemBackends lets a user-managed backend hide a same-named system backend", func() {
|
||||
must := func(err error) { Expect(err).NotTo(HaveOccurred()) }
|
||||
|
||||
mkBackend := func(root, name, alias string) string {
|
||||
dir := filepath.Join(root, name)
|
||||
must(os.MkdirAll(dir, 0o750))
|
||||
b, _ := json.Marshal(&BackendMetadata{Alias: alias, Name: name})
|
||||
must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644))
|
||||
must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755))
|
||||
return dir
|
||||
}
|
||||
|
||||
sysRoot, err := os.MkdirTemp("", "system-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(sysRoot)
|
||||
managedRoot, err := os.MkdirTemp("", "managed-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(managedRoot)
|
||||
|
||||
// The SAME concrete name exists in both roots.
|
||||
mkBackend(sysRoot, "cpu-audio-cpp", "audio-cpp")
|
||||
managedDir := mkBackend(managedRoot, "cpu-audio-cpp", "audio-cpp")
|
||||
|
||||
must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "cpu"))
|
||||
defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }()
|
||||
|
||||
state, err := system.GetSystemState(
|
||||
system.WithBackendPath(managedRoot),
|
||||
system.WithBackendSystemPath(sysRoot),
|
||||
)
|
||||
must(err)
|
||||
|
||||
backs, err := ListSystemBackends(state)
|
||||
must(err)
|
||||
|
||||
// Concrete name and alias BOTH run the managed installation and
|
||||
// report its metadata — never the system executable with managed
|
||||
// metadata (or any other cross-root mix).
|
||||
concrete, ok := backs.Get("cpu-audio-cpp")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(concrete.RunFile).To(Equal(filepath.Join(managedDir, "run.sh")))
|
||||
Expect(concrete.IsSystem).To(BeFalse())
|
||||
|
||||
aliasBack, ok := backs.Get("audio-cpp")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(aliasBack.RunFile).To(Equal(filepath.Join(managedDir, "run.sh")))
|
||||
Expect(aliasBack.IsSystem).To(BeFalse())
|
||||
Expect(aliasBack.Metadata).NotTo(BeNil())
|
||||
Expect(aliasBack.Metadata.Name).To(Equal("cpu-audio-cpp"))
|
||||
})
|
||||
|
||||
It("ListSystemBackends preserves managed meta backends against system aliases", func() {
|
||||
sysRoot := filepath.Join(tempDir, "system")
|
||||
managedRoot := filepath.Join(tempDir, "managed")
|
||||
for _, dir := range []string{
|
||||
filepath.Join(sysRoot, "cpu-audio-cpp"),
|
||||
filepath.Join(managedRoot, "custom-audio"),
|
||||
filepath.Join(managedRoot, "audio-cpp"),
|
||||
} {
|
||||
Expect(os.MkdirAll(dir, 0o750)).To(Succeed())
|
||||
}
|
||||
Expect(os.WriteFile(filepath.Join(sysRoot, "cpu-audio-cpp", "run.sh"), nil, 0o755)).To(Succeed())
|
||||
managedRun := filepath.Join(managedRoot, "custom-audio", "run.sh")
|
||||
Expect(os.WriteFile(managedRun, nil, 0o755)).To(Succeed())
|
||||
Expect(writeBackendMetadata(filepath.Join(sysRoot, "cpu-audio-cpp"), &BackendMetadata{
|
||||
Name: "cpu-audio-cpp", Alias: "audio-cpp",
|
||||
})).To(Succeed())
|
||||
Expect(writeBackendMetadata(filepath.Join(managedRoot, "audio-cpp"), &BackendMetadata{
|
||||
Name: "audio-cpp", MetaBackendFor: "custom-audio",
|
||||
})).To(Succeed())
|
||||
state, err := system.GetSystemState(
|
||||
system.WithBackendPath(managedRoot),
|
||||
system.WithBackendSystemPath(sysRoot),
|
||||
)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
backends, err := ListSystemBackends(state)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
backend, ok := backends.Get("audio-cpp")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(backend.RunFile).To(Equal(managedRun))
|
||||
Expect(backend.IsSystem).To(BeFalse())
|
||||
Expect(backend.IsMeta).To(BeTrue())
|
||||
Expect(backend.Metadata.MetaBackendFor).To(Equal("custom-audio"))
|
||||
})
|
||||
|
||||
It("ListSystemBackends lets a user-managed variant take over its whole alias group", func() {
|
||||
must := func(err error) { Expect(err).NotTo(HaveOccurred()) }
|
||||
|
||||
mkBackend := func(root, name, alias string) string {
|
||||
dir := filepath.Join(root, name)
|
||||
must(os.MkdirAll(dir, 0o750))
|
||||
b, _ := json.Marshal(&BackendMetadata{Alias: alias, Name: name})
|
||||
must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644))
|
||||
must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755))
|
||||
return dir
|
||||
}
|
||||
|
||||
sysRoot, err := os.MkdirTemp("", "system-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(sysRoot)
|
||||
managedRoot, err := os.MkdirTemp("", "managed-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(managedRoot)
|
||||
|
||||
// System family with the variant an NVIDIA host would prefer;
|
||||
// the user installs only the cpu variant via the gallery.
|
||||
mkBackend(sysRoot, "cuda12-audio-cpp", "audio-cpp")
|
||||
mkBackend(sysRoot, "vulkan-audio-cpp", "audio-cpp")
|
||||
managedDir := mkBackend(managedRoot, "cpu-audio-cpp", "audio-cpp")
|
||||
|
||||
must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia"))
|
||||
defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }()
|
||||
|
||||
state, err := system.GetSystemState(
|
||||
system.WithBackendPath(managedRoot),
|
||||
system.WithBackendSystemPath(sysRoot),
|
||||
)
|
||||
must(err)
|
||||
state.GPUVendor = "nvidia"
|
||||
|
||||
backs, err := ListSystemBackends(state)
|
||||
must(err)
|
||||
|
||||
// The alias resolves to the managed variant even though the
|
||||
// capability tokens would prefer the system cuda build: a
|
||||
// gallery-installed variant takes over its whole family.
|
||||
aliasBack, ok := backs.Get("audio-cpp")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(aliasBack.RunFile).To(Equal(filepath.Join(managedDir, "run.sh")))
|
||||
Expect(aliasBack.IsSystem).To(BeFalse())
|
||||
|
||||
// The system family is invisible entirely, concrete names
|
||||
// included — a stale system variant may not work with newer
|
||||
// models, so it must not stay reachable.
|
||||
_, ok = backs.Get("cuda12-audio-cpp")
|
||||
Expect(ok).To(BeFalse())
|
||||
_, ok = backs.Get("vulkan-audio-cpp")
|
||||
Expect(ok).To(BeFalse())
|
||||
})
|
||||
|
||||
It("ListSystemBackends resolves a mixed population across both roots", func() {
|
||||
must := func(err error) { Expect(err).NotTo(HaveOccurred()) }
|
||||
|
||||
// alias == "" installs no metadata.json at all (old-style dir).
|
||||
mkBackend := func(root, name, alias string) string {
|
||||
dir := filepath.Join(root, name)
|
||||
must(os.MkdirAll(dir, 0o750))
|
||||
if alias != "" {
|
||||
b, _ := json.Marshal(&BackendMetadata{Alias: alias, Name: name})
|
||||
must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644))
|
||||
}
|
||||
must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755))
|
||||
return dir
|
||||
}
|
||||
|
||||
sysRoot, err := os.MkdirTemp("", "system-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(sysRoot)
|
||||
managedRoot, err := os.MkdirTemp("", "managed-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(managedRoot)
|
||||
|
||||
// System root: two aliased families and a metadata-less loner.
|
||||
mkBackend(sysRoot, "cpu-audio-cpp", "audio-cpp")
|
||||
mkBackend(sysRoot, "cuda12-audio-cpp", "audio-cpp")
|
||||
mkBackend(sysRoot, "cpu-llama-cpp", "llama-cpp")
|
||||
sysVulkanLlama := mkBackend(sysRoot, "vulkan-llama-cpp", "llama-cpp")
|
||||
mkBackend(sysRoot, "piper", "")
|
||||
|
||||
// Managed root: a family takeover, a metadata-less name
|
||||
// collision, a metadata-less variant that must NOT take over
|
||||
// its would-be family, a plain loner, and its own family.
|
||||
managedCPUAudio := mkBackend(managedRoot, "cpu-audio-cpp", "audio-cpp")
|
||||
managedWhisper := mkBackend(managedRoot, "whisper", "")
|
||||
mkBackend(sysRoot, "whisper", "")
|
||||
mkBackend(managedRoot, "rocm-llama-cpp", "")
|
||||
mkBackend(managedRoot, "bark", "")
|
||||
managedKokoro := mkBackend(managedRoot, "kokoro", "tts-suite")
|
||||
|
||||
must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia"))
|
||||
defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }()
|
||||
|
||||
state, err := system.GetSystemState(
|
||||
system.WithBackendPath(managedRoot),
|
||||
system.WithBackendSystemPath(sysRoot),
|
||||
)
|
||||
must(err)
|
||||
state.GPUVendor = "nvidia"
|
||||
|
||||
backs, err := ListSystemBackends(state)
|
||||
must(err)
|
||||
|
||||
expectBackend := func(name, runFile string, isSystem bool) {
|
||||
b, ok := backs.Get(name)
|
||||
Expect(ok).To(BeTrue(), name)
|
||||
Expect(b.RunFile).To(Equal(runFile), name)
|
||||
Expect(b.IsSystem).To(Equal(isSystem), name)
|
||||
}
|
||||
expectGone := func(name string) {
|
||||
_, ok := backs.Get(name)
|
||||
Expect(ok).To(BeFalse(), name)
|
||||
}
|
||||
|
||||
// audio-cpp family: the managed cpu variant takes over — the
|
||||
// alias resolves to it despite the nvidia tokens preferring
|
||||
// cuda, and the system family is gone, concretes included.
|
||||
expectBackend("audio-cpp", filepath.Join(managedCPUAudio, "run.sh"), false)
|
||||
expectBackend("cpu-audio-cpp", filepath.Join(managedCPUAudio, "run.sh"), false)
|
||||
expectGone("cuda12-audio-cpp")
|
||||
|
||||
// llama-cpp family: the managed rocm-llama-cpp declares NO
|
||||
// metadata, so it joins no family and triggers no takeover —
|
||||
// the system family stays and resolves by capability (vulkan
|
||||
// on this nvidia host, no cuda variant present).
|
||||
expectBackend("llama-cpp", filepath.Join(sysVulkanLlama, "run.sh"), true)
|
||||
expectBackend("cpu-llama-cpp", filepath.Join(sysRoot, "cpu-llama-cpp", "run.sh"), true)
|
||||
expectBackend("vulkan-llama-cpp", filepath.Join(sysVulkanLlama, "run.sh"), true)
|
||||
expectBackend("rocm-llama-cpp", filepath.Join(managedRoot, "rocm-llama-cpp", "run.sh"), false)
|
||||
|
||||
// whisper: same metadata-less name in both roots — managed
|
||||
// hides system.
|
||||
expectBackend("whisper", filepath.Join(managedWhisper, "run.sh"), false)
|
||||
|
||||
// Loners survive untouched on their own side.
|
||||
expectBackend("piper", filepath.Join(sysRoot, "piper", "run.sh"), true)
|
||||
expectBackend("bark", filepath.Join(managedRoot, "bark", "run.sh"), false)
|
||||
|
||||
// A managed-only alias group resolves within itself.
|
||||
expectBackend("tts-suite", filepath.Join(managedKokoro, "run.sh"), false)
|
||||
expectBackend("kokoro", filepath.Join(managedKokoro, "run.sh"), false)
|
||||
})
|
||||
|
||||
It("ListSystemBackends: name collisions across roots, all metadata permutations", func() {
|
||||
must := func(err error) { Expect(err).NotTo(HaveOccurred()) }
|
||||
|
||||
// withMeta installs metadata.json; alias may be empty (name-only
|
||||
// metadata — possible and legal, e.g. a packaged backend that
|
||||
// declares no family).
|
||||
mk := func(root, name, alias string, withMeta bool) string {
|
||||
dir := filepath.Join(root, name)
|
||||
must(os.MkdirAll(dir, 0o750))
|
||||
if withMeta {
|
||||
b, _ := json.Marshal(&BackendMetadata{Alias: alias, Name: name})
|
||||
must(os.WriteFile(filepath.Join(dir, "metadata.json"), b, 0o644))
|
||||
}
|
||||
must(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(""), 0o755))
|
||||
return dir
|
||||
}
|
||||
|
||||
sysRoot, err := os.MkdirTemp("", "system-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(sysRoot)
|
||||
managedRoot, err := os.MkdirTemp("", "managed-backends-*")
|
||||
must(err)
|
||||
defer os.RemoveAll(managedRoot)
|
||||
|
||||
// 1a. Metadata on BOTH sides, aliased side managed:
|
||||
// system concrete alpha (name-only metadata) vs managed
|
||||
// cuda-alpha aliased to alpha.
|
||||
mk(sysRoot, "alpha", "", true)
|
||||
cudaAlpha := mk(managedRoot, "cuda-alpha", "alpha", true)
|
||||
|
||||
// 1b. Reverse: managed concrete beta (name-only metadata) vs
|
||||
// system cuda-beta aliased to beta.
|
||||
beta := mk(managedRoot, "beta", "", true)
|
||||
mk(sysRoot, "cuda-beta", "beta", true)
|
||||
|
||||
// 2a. No metadata on ONE side (system): bare system gamma vs
|
||||
// managed cuda-gamma aliased to gamma.
|
||||
mk(sysRoot, "gamma", "", false)
|
||||
cudaGamma := mk(managedRoot, "cuda-gamma", "gamma", true)
|
||||
|
||||
// 2b. Reverse: bare managed delta vs system cuda-delta aliased
|
||||
// to delta.
|
||||
delta := mk(managedRoot, "delta", "", false)
|
||||
mk(sysRoot, "cuda-delta", "delta", true)
|
||||
|
||||
// 3a. Same name in BOTH roots, metadata on both.
|
||||
epsilonManaged := mk(managedRoot, "epsilon", "", true)
|
||||
mk(sysRoot, "epsilon", "", true)
|
||||
|
||||
// 3b. Same name in BOTH roots, metadata on neither.
|
||||
zetaManaged := mk(managedRoot, "zeta", "", false)
|
||||
mk(sysRoot, "zeta", "", false)
|
||||
|
||||
must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia"))
|
||||
defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }()
|
||||
|
||||
state, err := system.GetSystemState(
|
||||
system.WithBackendPath(managedRoot),
|
||||
system.WithBackendSystemPath(sysRoot),
|
||||
)
|
||||
must(err)
|
||||
state.GPUVendor = "nvidia"
|
||||
|
||||
backs, err := ListSystemBackends(state)
|
||||
must(err)
|
||||
|
||||
expect := func(name, runFile string, isSystem bool) {
|
||||
b, ok := backs.Get(name)
|
||||
Expect(ok).To(BeTrue(), name)
|
||||
Expect(b.RunFile).To(Equal(filepath.Join(runFile, "run.sh")), name)
|
||||
Expect(b.IsSystem).To(Equal(isSystem), name)
|
||||
}
|
||||
|
||||
// 1a: the managed alias owns the name; the system concrete
|
||||
// (metadata or not) is unreachable.
|
||||
expect("alpha", cudaAlpha, false)
|
||||
expect("cuda-alpha", cudaAlpha, false)
|
||||
|
||||
// 1b: the managed concrete keeps its name — a system alias
|
||||
// never hijacks it; the system variant stays concrete-only.
|
||||
expect("beta", beta, false)
|
||||
b, ok := backs.Get("cuda-beta")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(b.IsSystem).To(BeTrue())
|
||||
|
||||
// 2a: same as 1a — the system concrete's missing metadata
|
||||
// changes nothing.
|
||||
expect("gamma", cudaGamma, false)
|
||||
|
||||
// 2b: same as 1b — the managed concrete's missing metadata
|
||||
// changes nothing.
|
||||
expect("delta", delta, false)
|
||||
b, ok = backs.Get("cuda-delta")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(b.IsSystem).To(BeTrue())
|
||||
|
||||
// 3a/3b: plain same-name hiding, managed wins, with or
|
||||
// without metadata on either side.
|
||||
expect("epsilon", epsilonManaged, false)
|
||||
expect("zeta", zetaManaged, false)
|
||||
})
|
||||
|
||||
It("ListSystemBackends prefers optimal alias candidate", func() {
|
||||
// Arrange two installed backends sharing the same alias
|
||||
must := func(err error) { Expect(err).NotTo(HaveOccurred()) }
|
||||
|
||||
@@ -148,6 +148,75 @@ export LOCALAI_EXTERNAL_BACKENDS="llm-backend,diffusion-backend"
|
||||
local-ai run
|
||||
```
|
||||
|
||||
## Backend Directory Format
|
||||
|
||||
Every backend, whether the gallery installed it into the user-managed
|
||||
location or a system package shipped it, is a directory with one
|
||||
required file:
|
||||
|
||||
- `run.sh` — the entry point LocalAI executes to start the backend.
|
||||
|
||||
and one optional file, `metadata.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "rocm-audio-cpp",
|
||||
"alias": "audio-cpp"
|
||||
}
|
||||
```
|
||||
|
||||
- `name` — the concrete backend name (defaults to the directory name).
|
||||
- `alias` — registers this directory as a *variant* of a backend
|
||||
family. When several installed variants share an alias
|
||||
(`cpu-audio-cpp`, `rocm-audio-cpp`, ... all aliased to `audio-cpp`),
|
||||
a model config using `backend: audio-cpp` resolves to the variant
|
||||
best matching the host's capability (CUDA before Vulkan before CPU
|
||||
on an NVIDIA host, ROCm first on AMD, and so on). Each variant also
|
||||
stays individually addressable by its concrete name, e.g.
|
||||
`backend: cpu-audio-cpp` to keep VRAM free for other models.
|
||||
- `meta_backend_for` — points a meta entry at a concrete backend
|
||||
directory installed next to it.
|
||||
|
||||
A directory without `metadata.json` is a plain backend under its
|
||||
directory name. Gallery installs write this metadata automatically
|
||||
(with additional bookkeeping fields such as `gallery_url` and
|
||||
`installed_at`); it only needs writing by hand when packaging backends
|
||||
outside the gallery.
|
||||
|
||||
## System-Provided Backends
|
||||
|
||||
Backends do not have to come from the gallery: directories under
|
||||
`LOCALAI_BACKENDS_SYSTEM_PATH` (default `/var/lib/local-ai/backends`)
|
||||
are discovered on every scan, using the same
|
||||
[directory format](#backend-directory-format) as user-managed
|
||||
backends. This is the integration point for distribution packages —
|
||||
the package manager installs backends there, while gallery installs
|
||||
keep living in the user-managed `LOCALAI_BACKENDS_PATH`.
|
||||
|
||||
One difference in error handling: a system directory with unreadable
|
||||
metadata is skipped with a warning, while unreadable metadata in the
|
||||
user-managed location fails the listing — a system package must never
|
||||
be able to break the discovery of the user's own backends.
|
||||
|
||||
### Precedence between the two locations
|
||||
|
||||
User-managed backends always win over system-provided ones:
|
||||
|
||||
- **Same name in both locations** — the user-managed backend hides the
|
||||
system one entirely.
|
||||
- **Family takeover** — installing *any* variant of an alias family
|
||||
into the user-managed location (e.g. from the gallery) replaces the
|
||||
whole system family: the alias resolves only among user-managed
|
||||
variants, and the system family's concrete names disappear from the
|
||||
listing. Variants of one family are versioned together; resolution
|
||||
never mixes installations of different origins within a family, and
|
||||
a stale system variant is not kept reachable.
|
||||
- **Names never get hijacked** — a system variant's alias cannot take
|
||||
over a name that exists as a user-managed backend (including a meta
|
||||
backend): `backend:
|
||||
audio-cpp` keeps running the user's `audio-cpp` installation even if
|
||||
a system package later ships variants aliased to that name.
|
||||
|
||||
## Creating a Backend
|
||||
|
||||
To create a new backend, you need to:
|
||||
|
||||
Reference in new issue
Block a user