From 673a80a57879b6dec1c71f2488da9783fc9b5736 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 23:34:01 +0100 Subject: [PATCH] feat: Filter backend gallery by system capabilities (#7950) * Initial plan * Add backend gallery filtering based on system capabilities Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Refactor L4T backend check to come before NVIDIA check Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Refactor: move capabilities business logic to capabilities.go and use constants Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * feat: display system capability in webui and refactor tests Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * chore: rename System/Capability Signed-off-by: Ettore Di Giacinto * refactor: use getSystemCapabilities in IsBackendCompatible for consistency Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * refactor: keep unused constants private in capabilities.go Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * fix: skip AMD/ROCm and Intel/SYCL tests on darwin Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- core/gallery/backend_types.go | 19 +++ core/gallery/backends_test.go | 246 ++++++++++++++++++++++++++++++++++ core/gallery/gallery.go | 22 ++- core/http/routes/ui_api.go | 7 + core/http/views/backends.html | 7 + pkg/system/capabilities.go | 98 +++++++++++--- 6 files changed, 383 insertions(+), 16 deletions(-) diff --git a/core/gallery/backend_types.go b/core/gallery/backend_types.go index acb7d5327..0fb6e7f24 100644 --- a/core/gallery/backend_types.go +++ b/core/gallery/backend_types.go @@ -63,6 +63,25 @@ func (m *GalleryBackend) IsMeta() bool { return len(m.CapabilitiesMap) > 0 && m.URI == "" } +// IsCompatibleWith checks if the backend is compatible with the current system capability. +// For meta backends, it checks if any of the capabilities in the map match the system capability. +// For concrete backends, it delegates to SystemState.IsBackendCompatible. +func (m *GalleryBackend) IsCompatibleWith(systemState *system.SystemState) bool { + if systemState == nil { + return true + } + + // Meta backends are compatible if the system capability matches one of the keys + if m.IsMeta() { + capability := systemState.Capability(m.CapabilitiesMap) + _, exists := m.CapabilitiesMap[capability] + return exists + } + + // For concrete backends, delegate to the system package + return systemState.IsBackendCompatible(m.Name, m.URI) +} + func (m *GalleryBackend) SetInstalled(installed bool) { m.Installed = installed } diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index 3799dc682..96ffe0fe5 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -172,6 +172,252 @@ var _ = Describe("Gallery Backends", func() { Expect(nilMetaBackend.IsMeta()).To(BeFalse()) }) + It("should check IsCompatibleWith correctly for meta backends", func() { + metaBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "meta-backend", + }, + CapabilitiesMap: map[string]string{ + "nvidia": "nvidia-backend", + "amd": "amd-backend", + "default": "default-backend", + }, + } + + // Test with nil state - should be compatible + Expect(metaBackend.IsCompatibleWith(nil)).To(BeTrue()) + + // Test with NVIDIA system - should be compatible (has nvidia key) + nvidiaState := &system.SystemState{GPUVendor: "nvidia", VRAM: 8 * 1024 * 1024 * 1024} + Expect(metaBackend.IsCompatibleWith(nvidiaState)).To(BeTrue()) + + // Test with default (no GPU) - should be compatible (has default key) + defaultState := &system.SystemState{} + Expect(metaBackend.IsCompatibleWith(defaultState)).To(BeTrue()) + }) + + Describe("IsCompatibleWith for concrete backends", func() { + Context("CPU backends", func() { + It("should be compatible on all systems", func() { + cpuBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "cpu-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-cpu-llama-cpp", + } + Expect(cpuBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue()) + Expect(cpuBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + Expect(cpuBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + }) + }) + + Context("Darwin/Metal backends", func() { + When("running on darwin", func() { + BeforeEach(func() { + if runtime.GOOS != "darwin" { + Skip("Skipping darwin-specific tests on non-darwin system") + } + }) + + It("should be compatible for MLX backend", func() { + mlxBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "mlx", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx", + } + Expect(mlxBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue()) + }) + + It("should be compatible for metal-llama-cpp backend", func() { + metalBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "metal-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-llama-cpp", + } + Expect(metalBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue()) + }) + }) + + When("running on non-darwin", func() { + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("Skipping non-darwin-specific tests on darwin system") + } + }) + + It("should NOT be compatible for MLX backend", func() { + mlxBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "mlx", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx", + } + Expect(mlxBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse()) + }) + + It("should NOT be compatible for metal-llama-cpp backend", func() { + metalBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "metal-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-llama-cpp", + } + Expect(metalBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse()) + }) + }) + }) + + Context("NVIDIA/CUDA backends", func() { + When("running on non-darwin", func() { + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("Skipping CUDA tests on darwin system") + } + }) + + It("should NOT be compatible without nvidia GPU", func() { + cudaBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "cuda12-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp", + } + Expect(cudaBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse()) + Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse()) + }) + + It("should be compatible with nvidia GPU", func() { + cudaBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "cuda12-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp", + } + Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + }) + + It("should be compatible with cuda13 backend on nvidia GPU", func() { + cuda13Backend := &GalleryBackend{ + Metadata: Metadata{ + Name: "cuda13-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-llama-cpp", + } + Expect(cuda13Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + }) + }) + }) + + Context("AMD/ROCm backends", func() { + When("running on non-darwin", func() { + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("Skipping AMD/ROCm tests on darwin system") + } + }) + + It("should NOT be compatible without AMD GPU", func() { + rocmBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "rocm-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp", + } + Expect(rocmBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse()) + Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse()) + }) + + It("should be compatible with AMD GPU", func() { + rocmBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "rocm-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp", + } + Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + }) + + It("should be compatible with hipblas backend on AMD GPU", func() { + hipBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "hip-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-hip-llama-cpp", + } + Expect(hipBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + }) + }) + }) + + Context("Intel/SYCL backends", func() { + When("running on non-darwin", func() { + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("Skipping Intel/SYCL tests on darwin system") + } + }) + + It("should NOT be compatible without Intel GPU", func() { + intelBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "intel-sycl-f16-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp", + } + Expect(intelBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse()) + Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse()) + }) + + It("should be compatible with Intel GPU", func() { + intelBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "intel-sycl-f16-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp", + } + Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + }) + + It("should be compatible with intel-sycl-f32 backend on Intel GPU", func() { + intelF32Backend := &GalleryBackend{ + Metadata: Metadata{ + Name: "intel-sycl-f32-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-llama-cpp", + } + Expect(intelF32Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + }) + + It("should be compatible with intel-transformers backend on Intel GPU", func() { + intelTransformersBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "intel-transformers", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-intel-transformers", + } + Expect(intelTransformersBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue()) + }) + }) + }) + + Context("Vulkan backends", func() { + It("should be compatible on CPU-only systems", func() { + // Vulkan backends don't have a specific GPU vendor requirement in the current logic + // They are compatible if no other GPU-specific pattern matches + vulkanBackend := &GalleryBackend{ + Metadata: Metadata{ + Name: "vulkan-llama-cpp", + }, + URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-llama-cpp", + } + // Vulkan doesn't have vendor-specific filtering in current implementation + Expect(vulkanBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue()) + }) + }) + }) + It("should find best backend from meta based on system capabilities", func() { metaBackend := &GalleryBackend{ diff --git a/core/gallery/gallery.go b/core/gallery/gallery.go index a3eb0f772..6add8cfa7 100644 --- a/core/gallery/gallery.go +++ b/core/gallery/gallery.go @@ -226,6 +226,16 @@ func AvailableGalleryModels(galleries []config.Gallery, systemState *system.Syst // List available backends func AvailableBackends(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) { + return availableBackendsWithFilter(galleries, systemState, true) +} + +// AvailableBackendsUnfiltered returns all available backends without filtering by system capability. +func AvailableBackendsUnfiltered(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) { + return availableBackendsWithFilter(galleries, systemState, false) +} + +// availableBackendsWithFilter is a helper function that lists available backends with optional filtering. +func availableBackendsWithFilter(galleries []config.Gallery, systemState *system.SystemState, filterByCapability bool) (GalleryElements[*GalleryBackend], error) { var backends []*GalleryBackend systemBackends, err := ListSystemBackends(systemState) @@ -241,7 +251,17 @@ func AvailableBackends(galleries []config.Gallery, systemState *system.SystemSta if err != nil { return nil, err } - backends = append(backends, galleryBackends...) + + // Filter backends by system capability if requested + if filterByCapability { + for _, backend := range galleryBackends { + if backend.IsCompatibleWith(systemState) { + backends = append(backends, backend) + } + } + } else { + backends = append(backends, galleryBackends...) + } } return backends, nil diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index 78b19468f..31dd66e1f 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -617,6 +617,12 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model installedBackendsCount = len(installedBackends) } + // Get the detected system capability + detectedCapability := "" + if appConfig.SystemState != nil { + detectedCapability = appConfig.SystemState.DetectedCapability() + } + return c.JSON(200, map[string]interface{}{ "backends": backendsJSON, "repositories": appConfig.BackendGalleries, @@ -629,6 +635,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "totalPages": totalPages, "prevPage": prevPage, "nextPage": nextPage, + "systemCapability": detectedCapability, }) }) diff --git a/core/http/views/backends.html b/core/http/views/backends.html index 4c5aa51d6..f48132b45 100644 --- a/core/http/views/backends.html +++ b/core/http/views/backends.html @@ -54,6 +54,11 @@ installed +
+ + Capability: + +
Documentation @@ -588,6 +593,7 @@ function backendsGallery() { totalPages: 1, availableBackends: 0, installedBackends: 0, + systemCapability: '', selectedBackend: null, jobProgress: {}, notifications: [], @@ -683,6 +689,7 @@ function backendsGallery() { this.totalPages = data.totalPages || 1; this.availableBackends = data.availableBackends || 0; this.installedBackends = data.installedBackends || 0; + this.systemCapability = data.systemCapability || 'default'; } catch (error) { console.error('Error fetching backends:', error); } finally { diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 9984dce62..60e05a3e8 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -12,15 +12,17 @@ import ( ) const ( + // Public constants - used by tests and external packages + Nvidia = "nvidia" + AMD = "amd" + Intel = "intel" + + // Private constants - only used within this package defaultCapability = "default" nvidiaL4T = "nvidia-l4t" darwinX86 = "darwin-x86" metal = "metal" - nvidia = "nvidia" - - amd = "amd" - intel = "intel" - vulkan = "vulkan" + vulkan = "vulkan" nvidiaCuda13 = "nvidia-cuda-13" nvidiaCuda12 = "nvidia-cuda-12" @@ -30,6 +32,16 @@ const ( capabilityEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY" capabilityRunFileEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY_RUN_FILE" defaultRunFile = "/run/localai/capability" + + // Backend detection tokens (private) + backendTokenDarwin = "darwin" + backendTokenMLX = "mlx" + backendTokenMetal = "metal" + backendTokenL4T = "l4t" + backendTokenCUDA = "cuda" + backendTokenROCM = "rocm" + backendTokenHIP = "hip" + backendTokenSYCL = "sycl" ) var ( @@ -96,7 +108,7 @@ func (s *SystemState) getSystemCapabilities() string { // If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { - if s.GPUVendor == nvidia { + if s.GPUVendor == Nvidia { xlog.Info("Using nvidia-l4t capability (arm64 on linux)", "env", capabilityEnv) if cuda13DirExists { return nvidiaL4TCuda13 @@ -131,7 +143,6 @@ func (s *SystemState) getSystemCapabilities() string { return s.GPUVendor } - // BackendPreferenceTokens returns a list of substrings that represent the preferred // backend implementation order for the current system capability. Callers can use // these tokens to select the most appropriate concrete backend among multiple @@ -139,19 +150,76 @@ func (s *SystemState) getSystemCapabilities() string { func (s *SystemState) BackendPreferenceTokens() []string { capStr := strings.ToLower(s.getSystemCapabilities()) switch { - case strings.HasPrefix(capStr, nvidia): - return []string{"cuda", "vulkan", "cpu"} - case strings.HasPrefix(capStr, amd): - return []string{"rocm", "hip", "vulkan", "cpu"} - case strings.HasPrefix(capStr, intel): - return []string{"sycl", intel, "cpu"} + case strings.HasPrefix(capStr, Nvidia): + return []string{backendTokenCUDA, vulkan, "cpu"} + case strings.HasPrefix(capStr, AMD): + return []string{backendTokenROCM, backendTokenHIP, vulkan, "cpu"} + case strings.HasPrefix(capStr, Intel): + return []string{backendTokenSYCL, Intel, "cpu"} case strings.HasPrefix(capStr, metal): - return []string{"metal", "cpu"} + return []string{backendTokenMetal, "cpu"} case strings.HasPrefix(capStr, darwinX86): return []string{"darwin-x86", "cpu"} case strings.HasPrefix(capStr, vulkan): - return []string{"vulkan", "cpu"} + return []string{vulkan, "cpu"} default: return []string{"cpu"} } } + +// DetectedCapability returns the detected system capability string. +// This can be used by the UI to display what capability was detected. +func (s *SystemState) DetectedCapability() string { + return s.getSystemCapabilities() +} + +// IsBackendCompatible checks if a backend (identified by name and URI) is compatible +// with the current system capability. This function uses getSystemCapabilities to ensure +// consistency with capability detection (including VRAM checks, environment overrides, etc.). +func (s *SystemState) IsBackendCompatible(name, uri string) bool { + combined := strings.ToLower(name + " " + uri) + capability := s.getSystemCapabilities() + + // Check for darwin/macOS-specific backends (mlx, metal, darwin) + isDarwinBackend := strings.Contains(combined, backendTokenDarwin) || + strings.Contains(combined, backendTokenMLX) || + strings.Contains(combined, backendTokenMetal) + if isDarwinBackend { + // Darwin backends require the system to be running on darwin with metal or darwin-x86 capability + return capability == metal || capability == darwinX86 + } + + // Check for NVIDIA L4T-specific backends (arm64 Linux with NVIDIA GPU) + // This must be checked before the general NVIDIA check as L4T backends + // may also contain "cuda" or "nvidia" in their names + isL4TBackend := strings.Contains(combined, backendTokenL4T) + if isL4TBackend { + return strings.HasPrefix(capability, nvidiaL4T) + } + + // Check for NVIDIA/CUDA-specific backends (non-L4T) + isNvidiaBackend := strings.Contains(combined, backendTokenCUDA) || + strings.Contains(combined, Nvidia) + if isNvidiaBackend { + // NVIDIA backends are compatible with nvidia, nvidia-cuda-12, nvidia-cuda-13, and l4t capabilities + return strings.HasPrefix(capability, Nvidia) + } + + // Check for AMD/ROCm-specific backends + isAMDBackend := strings.Contains(combined, backendTokenROCM) || + strings.Contains(combined, backendTokenHIP) || + strings.Contains(combined, AMD) + if isAMDBackend { + return capability == AMD + } + + // Check for Intel/SYCL-specific backends + isIntelBackend := strings.Contains(combined, backendTokenSYCL) || + strings.Contains(combined, Intel) + if isIntelBackend { + return capability == Intel + } + + // CPU backends are always compatible + return true +}