mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 18:09:05 -04:00
b19afb192a
* fix(backends): list backends runnable on worker nodes in distributed mode GET /backends/available filtered the gallery against the system state of the host serving the request. In a distributed deployment that host is the controller, which typically has no GPU, while the GPUs live on worker nodes. Any meta backend whose capabilities map lacks a "default" (or "cpu") key was therefore dropped from the listing entirely — longcat-video, vllm-omni, ltx-video, parakeet, edgetam and qwentts were invisible in the UI even though installing them by name on a GPU worker worked fine. Workers now report their own meta-backend capability at registration and the controller persists it on the node row. The controller cannot derive it: OS-dependent capabilities (metal, darwin-x86, nvidia-l4t) and the CUDA runtime refinements are only observable on the worker. Nodes registered before this field existed fall back to a coarse capability derived from their GPU vendor and VRAM. Backend discovery then evaluates compatibility as the union over healthy backend nodes, so a backend runnable on any node is offered while one no node can run stays hidden. Each remote capability is evaluated through a capability-pinned system state, otherwise a forced capability on the controller image (LOCALAI_FORCE_META_BACKEND_CAPABILITY or /run/localai/capability) would silently override every worker's verdict. With no registered nodes the listing is byte-for-byte what it was, so single-node deployments are unaffected. Also fixes the same-root-cause misclassification in /api/operations, which used the capability-filtered listing to decide whether an operation was a backend or a model install. A GPU-only backend installing on a worker is still a backend operation on the controller, so that lookup is now unfiltered. Assisted-by: Claude:claude-opus-4-8 golangci-lint Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(backends): union worker capabilities in backend discovery Implementation for the specs added in the previous commit, plus the two remaining discovery endpoints. Capability-filtered backend discovery evaluated compatibility against the system state of the host serving the request. In a distributed deployment that host is the controller, which typically has no GPU, while the GPUs live on worker nodes. Any meta backend whose capabilities map lacks a "default" (or "cpu") key was dropped entirely — longcat-video, vllm-omni, ltx-video, parakeet, edgetam and qwentts were invisible in the UI even though installing them by name on a GPU worker worked fine. Workers now report their own meta-backend capability at registration and the controller persists it on the node row. The controller cannot derive it: OS-dependent capabilities (metal, darwin-x86, nvidia-l4t) and the CUDA runtime refinements are only observable on the worker. Nodes registered before this field existed fall back to a coarse capability derived from their GPU vendor and VRAM. Discovery then evaluates compatibility as the union over healthy backend nodes, so a backend runnable on any node is offered while one no node can run stays hidden. Each remote capability is evaluated through a capability-pinned system state, otherwise a forced capability on the controller image (LOCALAI_FORCE_META_BACKEND_CAPABILITY or /run/localai/capability) would silently override every worker's verdict. With no registered nodes the listing is byte-for-byte what it was, so single-node deployments are unaffected. Four surfaces shared this root cause and are all routed through the same helper now: - GET /backends/available - GET /api/fine-tuning/backends - GET /api/quantization/backends - /api/operations backend-vs-model classification, which additionally had no reason to filter by capability at all: a GPU-only backend installing on a worker is still a backend operation on the controller, so that lookup is now unfiltered. Assisted-by: Claude:claude-opus-4-8 golangci-lint Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
59 lines
1.9 KiB
Go
59 lines
1.9 KiB
Go
package system
|
|
|
|
import (
|
|
"os"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = Describe("NewCapabilityState", func() {
|
|
It("reports exactly the supplied capability", func() {
|
|
Expect(NewCapabilityState("nvidia-l4t-cuda-13").DetectedCapability()).To(Equal("nvidia-l4t-cuda-13"))
|
|
})
|
|
|
|
It("ignores a forced capability meant for the local host", func() {
|
|
// A controller container image commonly pins its own capability via
|
|
// this env var (or /run/localai/capability). That must never override
|
|
// the capability we are evaluating on a remote worker's behalf.
|
|
orig, had := os.LookupEnv(capabilityEnv)
|
|
Expect(os.Setenv(capabilityEnv, "metal")).To(Succeed())
|
|
DeferCleanup(func() {
|
|
if had {
|
|
Expect(os.Setenv(capabilityEnv, orig)).To(Succeed())
|
|
return
|
|
}
|
|
Expect(os.Unsetenv(capabilityEnv)).To(Succeed())
|
|
})
|
|
|
|
Expect(NewCapabilityState("nvidia-cuda-13").DetectedCapability()).To(Equal("nvidia-cuda-13"))
|
|
})
|
|
|
|
It("applies the supplied state options", func() {
|
|
state := NewCapabilityState("nvidia", WithBackendPath("/some/backends"))
|
|
Expect(state.Backend.BackendsPath).To(Equal("/some/backends"))
|
|
})
|
|
|
|
It("honors the disable escape hatch when that is the pinned capability", func() {
|
|
Expect(NewCapabilityState(disableCapability).CapabilityFilterDisabled()).To(BeTrue())
|
|
})
|
|
})
|
|
|
|
var _ = Describe("CapabilityFromGPU", func() {
|
|
const eightGB = 8 * 1024 * 1024 * 1024
|
|
const twoGB = 2 * 1024 * 1024 * 1024
|
|
|
|
It("returns the vendor for a GPU with enough VRAM", func() {
|
|
Expect(CapabilityFromGPU(Nvidia, eightGB)).To(Equal(Nvidia))
|
|
Expect(CapabilityFromGPU(AMD, eightGB)).To(Equal(AMD))
|
|
})
|
|
|
|
It("returns default when no GPU is reported", func() {
|
|
Expect(CapabilityFromGPU("", eightGB)).To(Equal(defaultCapability))
|
|
})
|
|
|
|
It("returns default when VRAM is below the usable threshold", func() {
|
|
Expect(CapabilityFromGPU(Nvidia, twoGB)).To(Equal(defaultCapability))
|
|
})
|
|
})
|