mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-03-04 23:19:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: Filter backend gallery by system capabilities (#7950)

Browse Source 
* Initial plan

* Add backend gallery filtering based on system capabilities

Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>

* Refactor L4T backend check to come before NVIDIA check

Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>

* Refactor: move capabilities business logic to capabilities.go and use constants

Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>

* feat: display system capability in webui and refactor tests

Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>

* chore: rename System/Capability

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* refactor: use getSystemCapabilities in IsBackendCompatible for consistency

Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>

* refactor: keep unused constants private in capabilities.go

Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>

* fix: skip AMD/ROCm and Intel/SYCL tests on darwin

Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Copilot
2026-01-10 23:34:01 +01:00
committed by GitHub
parent 2554e9fabe
commit 673a80a578
6 changed files with 383 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
core/gallery/backend_types.go
View File

@@ -63,6 +63,25 @@ func (m *GalleryBackend) IsMeta() bool {
return len(m.CapabilitiesMap) > 0 && m.URI == ""
}
// IsCompatibleWith checks if the backend is compatible with the current system capability.
// For meta backends, it checks if any of the capabilities in the map match the system capability.
// For concrete backends, it delegates to SystemState.IsBackendCompatible.
func (m *GalleryBackend) IsCompatibleWith(systemState *system.SystemState) bool {
if systemState == nil {
return true
}
// Meta backends are compatible if the system capability matches one of the keys
if m.IsMeta() {
capability := systemState.Capability(m.CapabilitiesMap)
_, exists := m.CapabilitiesMap[capability]
return exists
}
// For concrete backends, delegate to the system package
return systemState.IsBackendCompatible(m.Name, m.URI)
}
func (m *GalleryBackend) SetInstalled(installed bool) {
m.Installed = installed
}

246
core/gallery/backends_test.go
View File

@@ -172,6 +172,252 @@ var _ = Describe("Gallery Backends", func() {
Expect(nilMetaBackend.IsMeta()).To(BeFalse())
})
It("should check IsCompatibleWith correctly for meta backends", func() {
metaBackend := &GalleryBackend{
Metadata: Metadata{
Name: "meta-backend",
},
CapabilitiesMap: map[string]string{
"nvidia": "nvidia-backend",
"amd": "amd-backend",
"default": "default-backend",
},
}
// Test with nil state - should be compatible
Expect(metaBackend.IsCompatibleWith(nil)).To(BeTrue())
// Test with NVIDIA system - should be compatible (has nvidia key)
nvidiaState := &system.SystemState{GPUVendor: "nvidia", VRAM: 8 * 1024 * 1024 * 1024}
Expect(metaBackend.IsCompatibleWith(nvidiaState)).To(BeTrue())
// Test with default (no GPU) - should be compatible (has default key)
defaultState := &system.SystemState{}
Expect(metaBackend.IsCompatibleWith(defaultState)).To(BeTrue())
})
Describe("IsCompatibleWith for concrete backends", func() {
Context("CPU backends", func() {
It("should be compatible on all systems", func() {
cpuBackend := &GalleryBackend{
Metadata: Metadata{
Name: "cpu-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-cpu-llama-cpp",
}
Expect(cpuBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue())
Expect(cpuBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
Expect(cpuBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
})
})
Context("Darwin/Metal backends", func() {
When("running on darwin", func() {
BeforeEach(func() {
if runtime.GOOS != "darwin" {
Skip("Skipping darwin-specific tests on non-darwin system")
}
})
It("should be compatible for MLX backend", func() {
mlxBackend := &GalleryBackend{
Metadata: Metadata{
Name: "mlx",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx",
}
Expect(mlxBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue())
})
It("should be compatible for metal-llama-cpp backend", func() {
metalBackend := &GalleryBackend{
Metadata: Metadata{
Name: "metal-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-llama-cpp",
}
Expect(metalBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue())
})
})
When("running on non-darwin", func() {
BeforeEach(func() {
if runtime.GOOS == "darwin" {
Skip("Skipping non-darwin-specific tests on darwin system")
}
})
It("should NOT be compatible for MLX backend", func() {
mlxBackend := &GalleryBackend{
Metadata: Metadata{
Name: "mlx",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx",
}
Expect(mlxBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
})
It("should NOT be compatible for metal-llama-cpp backend", func() {
metalBackend := &GalleryBackend{
Metadata: Metadata{
Name: "metal-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-llama-cpp",
}
Expect(metalBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
})
})
})
Context("NVIDIA/CUDA backends", func() {
When("running on non-darwin", func() {
BeforeEach(func() {
if runtime.GOOS == "darwin" {
Skip("Skipping CUDA tests on darwin system")
}
})
It("should NOT be compatible without nvidia GPU", func() {
cudaBackend := &GalleryBackend{
Metadata: Metadata{
Name: "cuda12-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp",
}
Expect(cudaBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse())
})
It("should be compatible with nvidia GPU", func() {
cudaBackend := &GalleryBackend{
Metadata: Metadata{
Name: "cuda12-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp",
}
Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
})
It("should be compatible with cuda13 backend on nvidia GPU", func() {
cuda13Backend := &GalleryBackend{
Metadata: Metadata{
Name: "cuda13-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-llama-cpp",
}
Expect(cuda13Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
})
})
})
Context("AMD/ROCm backends", func() {
When("running on non-darwin", func() {
BeforeEach(func() {
if runtime.GOOS == "darwin" {
Skip("Skipping AMD/ROCm tests on darwin system")
}
})
It("should NOT be compatible without AMD GPU", func() {
rocmBackend := &GalleryBackend{
Metadata: Metadata{
Name: "rocm-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp",
}
Expect(rocmBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse())
})
It("should be compatible with AMD GPU", func() {
rocmBackend := &GalleryBackend{
Metadata: Metadata{
Name: "rocm-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp",
}
Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
})
It("should be compatible with hipblas backend on AMD GPU", func() {
hipBackend := &GalleryBackend{
Metadata: Metadata{
Name: "hip-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-hip-llama-cpp",
}
Expect(hipBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
})
})
})
Context("Intel/SYCL backends", func() {
When("running on non-darwin", func() {
BeforeEach(func() {
if runtime.GOOS == "darwin" {
Skip("Skipping Intel/SYCL tests on darwin system")
}
})
It("should NOT be compatible without Intel GPU", func() {
intelBackend := &GalleryBackend{
Metadata: Metadata{
Name: "intel-sycl-f16-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp",
}
Expect(intelBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse())
})
It("should be compatible with Intel GPU", func() {
intelBackend := &GalleryBackend{
Metadata: Metadata{
Name: "intel-sycl-f16-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp",
}
Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
})
It("should be compatible with intel-sycl-f32 backend on Intel GPU", func() {
intelF32Backend := &GalleryBackend{
Metadata: Metadata{
Name: "intel-sycl-f32-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-llama-cpp",
}
Expect(intelF32Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
})
It("should be compatible with intel-transformers backend on Intel GPU", func() {
intelTransformersBackend := &GalleryBackend{
Metadata: Metadata{
Name: "intel-transformers",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-intel-transformers",
}
Expect(intelTransformersBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
})
})
})
Context("Vulkan backends", func() {
It("should be compatible on CPU-only systems", func() {
// Vulkan backends don't have a specific GPU vendor requirement in the current logic
// They are compatible if no other GPU-specific pattern matches
vulkanBackend := &GalleryBackend{
Metadata: Metadata{
Name: "vulkan-llama-cpp",
},
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-llama-cpp",
}
// Vulkan doesn't have vendor-specific filtering in current implementation
Expect(vulkanBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue())
})
})
})
It("should find best backend from meta based on system capabilities", func() {
metaBackend := &GalleryBackend{

22
core/gallery/gallery.go
View File

@@ -226,6 +226,16 @@ func AvailableGalleryModels(galleries []config.Gallery, systemState *system.Syst
// List available backends
func AvailableBackends(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) {
return availableBackendsWithFilter(galleries, systemState, true)
}
// AvailableBackendsUnfiltered returns all available backends without filtering by system capability.
func AvailableBackendsUnfiltered(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) {
return availableBackendsWithFilter(galleries, systemState, false)
}
// availableBackendsWithFilter is a helper function that lists available backends with optional filtering.
func availableBackendsWithFilter(galleries []config.Gallery, systemState *system.SystemState, filterByCapability bool) (GalleryElements[*GalleryBackend], error) {
var backends []*GalleryBackend
systemBackends, err := ListSystemBackends(systemState)
@@ -241,7 +251,17 @@ func AvailableBackends(galleries []config.Gallery, systemState *system.SystemSta
if err != nil {
return nil, err
}
backends = append(backends, galleryBackends...)
// Filter backends by system capability if requested
if filterByCapability {
for _, backend := range galleryBackends {
if backend.IsCompatibleWith(systemState) {
backends = append(backends, backend)
}
}
} else {
backends = append(backends, galleryBackends...)
}
}
return backends, nil

7
core/http/routes/ui_api.go
View File

@@ -617,6 +617,12 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
installedBackendsCount = len(installedBackends)
}
// Get the detected system capability
detectedCapability := ""
if appConfig.SystemState != nil {
detectedCapability = appConfig.SystemState.DetectedCapability()
}
return c.JSON(200, map[string]interface{}{
"backends": backendsJSON,
"repositories": appConfig.BackendGalleries,
@@ -629,6 +635,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"totalPages": totalPages,
"prevPage": prevPage,
"nextPage": nextPage,
"systemCapability": detectedCapability,
})
})

7
core/http/views/backends.html
View File

@@ -54,6 +54,11 @@
<span class="font-semibold text-cyan-300" x-text="installedBackends"></span>
<span class="text-[#94A3B8] ml-1">installed</span>
</a>
<div class="flex items-center bg-[#101827] rounded-lg px-4 py-2 border border-[#38BDF8]/30">
<i class="fas fa-microchip text-[#38BDF8] mr-2"></i>
<span class="text-[#94A3B8] mr-1">Capability:</span>
<span class="font-semibold text-[#38BDF8]" x-text="systemCapability"></span>
</div>
<a href="https://localai.io/backends/" target="_blank" class="btn-primary">
<i class="fas fa-info-circle mr-2"></i>
<span>Documentation</span>
@@ -588,6 +593,7 @@ function backendsGallery() {
totalPages: 1,
availableBackends: 0,
installedBackends: 0,
systemCapability: '',
selectedBackend: null,
jobProgress: {},
notifications: [],
@@ -683,6 +689,7 @@ function backendsGallery() {
this.totalPages = data.totalPages || 1;
this.availableBackends = data.availableBackends || 0;
this.installedBackends = data.installedBackends || 0;
this.systemCapability = data.systemCapability || 'default';
} catch (error) {
console.error('Error fetching backends:', error);
} finally {

98
pkg/system/capabilities.go
View File

@@ -12,15 +12,17 @@ import (
)
const (
// Public constants - used by tests and external packages
Nvidia = "nvidia"
AMD = "amd"
Intel = "intel"
// Private constants - only used within this package
defaultCapability = "default"
nvidiaL4T = "nvidia-l4t"
darwinX86 = "darwin-x86"
metal = "metal"
nvidia = "nvidia"
amd = "amd"
intel = "intel"
vulkan = "vulkan"
vulkan = "vulkan"
nvidiaCuda13 = "nvidia-cuda-13"
nvidiaCuda12 = "nvidia-cuda-12"
@@ -30,6 +32,16 @@ const (
capabilityEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY"
capabilityRunFileEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY_RUN_FILE"
defaultRunFile = "/run/localai/capability"
// Backend detection tokens (private)
backendTokenDarwin = "darwin"
backendTokenMLX = "mlx"
backendTokenMetal = "metal"
backendTokenL4T = "l4t"
backendTokenCUDA = "cuda"
backendTokenROCM = "rocm"
backendTokenHIP = "hip"
backendTokenSYCL = "sycl"
)
var (
@@ -96,7 +108,7 @@ func (s *SystemState) getSystemCapabilities() string {
// If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t
if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" {
if s.GPUVendor == nvidia {
if s.GPUVendor == Nvidia {
xlog.Info("Using nvidia-l4t capability (arm64 on linux)", "env", capabilityEnv)
if cuda13DirExists {
return nvidiaL4TCuda13
@@ -131,7 +143,6 @@ func (s *SystemState) getSystemCapabilities() string {
return s.GPUVendor
}
// BackendPreferenceTokens returns a list of substrings that represent the preferred
// backend implementation order for the current system capability. Callers can use
// these tokens to select the most appropriate concrete backend among multiple
@@ -139,19 +150,76 @@ func (s *SystemState) getSystemCapabilities() string {
func (s *SystemState) BackendPreferenceTokens() []string {
capStr := strings.ToLower(s.getSystemCapabilities())
switch {
case strings.HasPrefix(capStr, nvidia):
return []string{"cuda", "vulkan", "cpu"}
case strings.HasPrefix(capStr, amd):
return []string{"rocm", "hip", "vulkan", "cpu"}
case strings.HasPrefix(capStr, intel):
return []string{"sycl", intel, "cpu"}
case strings.HasPrefix(capStr, Nvidia):
return []string{backendTokenCUDA, vulkan, "cpu"}
case strings.HasPrefix(capStr, AMD):
return []string{backendTokenROCM, backendTokenHIP, vulkan, "cpu"}
case strings.HasPrefix(capStr, Intel):
return []string{backendTokenSYCL, Intel, "cpu"}
case strings.HasPrefix(capStr, metal):
return []string{"metal", "cpu"}
return []string{backendTokenMetal, "cpu"}
case strings.HasPrefix(capStr, darwinX86):
return []string{"darwin-x86", "cpu"}
case strings.HasPrefix(capStr, vulkan):
return []string{"vulkan", "cpu"}
return []string{vulkan, "cpu"}
default:
return []string{"cpu"}
}
}
// DetectedCapability returns the detected system capability string.
// This can be used by the UI to display what capability was detected.
func (s *SystemState) DetectedCapability() string {
return s.getSystemCapabilities()
}
// IsBackendCompatible checks if a backend (identified by name and URI) is compatible
// with the current system capability. This function uses getSystemCapabilities to ensure
// consistency with capability detection (including VRAM checks, environment overrides, etc.).
func (s *SystemState) IsBackendCompatible(name, uri string) bool {
combined := strings.ToLower(name + " " + uri)
capability := s.getSystemCapabilities()
// Check for darwin/macOS-specific backends (mlx, metal, darwin)
isDarwinBackend := strings.Contains(combined, backendTokenDarwin) ||
strings.Contains(combined, backendTokenMLX) ||
strings.Contains(combined, backendTokenMetal)
if isDarwinBackend {
// Darwin backends require the system to be running on darwin with metal or darwin-x86 capability
return capability == metal || capability == darwinX86
}
// Check for NVIDIA L4T-specific backends (arm64 Linux with NVIDIA GPU)
// This must be checked before the general NVIDIA check as L4T backends
// may also contain "cuda" or "nvidia" in their names
isL4TBackend := strings.Contains(combined, backendTokenL4T)
if isL4TBackend {
return strings.HasPrefix(capability, nvidiaL4T)
}
// Check for NVIDIA/CUDA-specific backends (non-L4T)
isNvidiaBackend := strings.Contains(combined, backendTokenCUDA) ||
strings.Contains(combined, Nvidia)
if isNvidiaBackend {
// NVIDIA backends are compatible with nvidia, nvidia-cuda-12, nvidia-cuda-13, and l4t capabilities
return strings.HasPrefix(capability, Nvidia)
}
// Check for AMD/ROCm-specific backends
isAMDBackend := strings.Contains(combined, backendTokenROCM) ||
strings.Contains(combined, backendTokenHIP) ||
strings.Contains(combined, AMD)
if isAMDBackend {
return capability == AMD
}
// Check for Intel/SYCL-specific backends
isIntelBackend := strings.Contains(combined, backendTokenSYCL) ||
strings.Contains(combined, Intel)
if isIntelBackend {
return capability == Intel
}
// CPU backends are always compatible
return true
}