mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-13 14:56:11 -04:00
mudler's LocalAI [bot]andEttore Di Giacinto
29899cd1e0
* fix(ui): move node labels into the scheduling selector field The scheduling page kept a node-label browser open above the rules whether or not anyone was writing one, while the field that actually needs labels, the rule's node selector, was two bare text inputs with no hint of what the cluster reports. The browser is gone. The selector's key input now completes against the label keys the cluster uses, and the value input offers only the values that key takes. The roster already loads for the page, so the suggestions cost no request, and a roster that fails to load costs the admin the hints and nothing else. Suggestions stay suggestions: a key no node reports yet still commits as typed, which is how an admin writes a rule before labelling the nodes for it. Assisted-by: Claude:claude-opus-5 golangci-lint eslint playwright Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(distributed): size model fit against the cluster, not the frontend The models page asked the frontend how much memory a model may occupy. In distributed mode the frontend is usually a GPU-less pod while every model runs on a worker, so a fleet of GPU nodes was told it could only run the smallest CPU build. The variant picker's fits flag and its auto-selection came from the same place, as did the hardware recommendations. The registry now reports the largest single healthy backend node. The largest node, not the fleet total: a model loads into one node, so four 16GB workers are not a home for a 40GB model. An operator-set VRAM budget caps a node's contribution, because the scheduler refuses a load above that ceiling anyway, and a GPU node beats a CPU node holding more system RAM. GET /api/resources and GET /api/models carry this as an additional cluster object. Their aggregate and ram fields keep reporting the frontend's own hardware, which is what the resource monitor shows. Variant selection judges backends against the union of the capabilities present in the cluster, the way backend discovery already did. Every path degrades to the local host: no cluster object in single-node mode, and none when the registry cannot be read, so a hiccup narrows the answer back to single-node behaviour rather than marking the whole catalog too large. The verdicts now name the node they belong to, since a model fits somewhere or nowhere. Assisted-by: Claude:claude-opus-5 golangci-lint eslint playwright Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
95 lines
4.0 KiB
Go
95 lines
4.0 KiB
Go
package gallery_test
|
|
|
|
import (
|
|
"context"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
|
|
"github.com/mudler/LocalAI/core/gallery"
|
|
"github.com/mudler/LocalAI/pkg/system"
|
|
)
|
|
|
|
// On a distributed controller the GPUs live on the workers, so a variant
|
|
// picker sized against the controller tells admins a cluster of A100s can only
|
|
// run the smallest CPU build.
|
|
var _ = Describe("ClusterResolveEnv", func() {
|
|
gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 }
|
|
|
|
// The controller as Argus actually runs it: no GPU at all.
|
|
var controller *system.SystemState
|
|
|
|
BeforeEach(func() {
|
|
controller = system.NewCapabilityState("default")
|
|
})
|
|
|
|
It("sizes models against the cluster reading rather than the controller", func() {
|
|
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
|
|
|
|
Expect(env.AvailableMemory).To(Equal(gib(80)))
|
|
})
|
|
|
|
It("accepts a CUDA backend that only the workers can run", func() {
|
|
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
|
|
|
|
Expect(env.BackendCompatible).ToNot(BeNil())
|
|
// A name carrying the cuda token is what the controller rejects today;
|
|
// a bare engine name like "vllm" passes on any host and would prove
|
|
// nothing about the union.
|
|
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
|
|
Expect(env.BackendCompatible("llama-cpp")).To(BeTrue())
|
|
})
|
|
|
|
// The union must stay a filter, not an open door: a Linux NVIDIA fleet
|
|
// still cannot run an Apple-only build.
|
|
It("still rejects a backend no node in the cluster can run", func() {
|
|
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
|
|
|
|
Expect(env.BackendCompatible("mlx")).To(BeFalse())
|
|
})
|
|
|
|
It("accepts a backend that any one node in a mixed fleet can run", func() {
|
|
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13", "metal"})
|
|
|
|
Expect(env.BackendCompatible("mlx")).To(BeTrue())
|
|
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
|
|
})
|
|
|
|
// Ranking has to follow the hardware too, or a cluster of NVIDIA workers
|
|
// gets offered the GGUF build over the vLLM one it should prefer.
|
|
It("ranks engines by the workers' hardware, not the controller's", func() {
|
|
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
|
|
|
|
Expect(env.EnginePreference).To(Equal(system.NewCapabilityState("nvidia-cuda-13").EnginePreferenceTokens()))
|
|
})
|
|
|
|
// Every degradation path lands here, so it must be indistinguishable from
|
|
// the single-node behavior that shipped before any of this existed.
|
|
It("falls back to the host description when the cluster reports nothing", func() {
|
|
host := gallery.HostResolveEnv(context.Background(), controller)
|
|
env := gallery.ClusterResolveEnv(context.Background(), controller, 0, nil)
|
|
|
|
Expect(env.AvailableMemory).To(Equal(host.AvailableMemory))
|
|
Expect(env.EnginePreference).To(Equal(host.EnginePreference))
|
|
Expect(env.BackendCompatible("cuda-13-vllm")).To(Equal(host.BackendCompatible("cuda-13-vllm")))
|
|
Expect(env.BackendCompatible("mlx")).To(Equal(host.BackendCompatible("mlx")))
|
|
})
|
|
|
|
// A cluster that reports capabilities but no usable memory reading should
|
|
// still gain the hardware view; only the size question falls back.
|
|
It("keeps the host memory when only the memory reading is missing", func() {
|
|
host := gallery.HostResolveEnv(context.Background(), controller)
|
|
env := gallery.ClusterResolveEnv(context.Background(), controller, 0, []string{"nvidia-cuda-13"})
|
|
|
|
Expect(env.AvailableMemory).To(Equal(host.AvailableMemory))
|
|
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
|
|
})
|
|
|
|
It("keeps the probe wired so variant sizes are still measured", func() {
|
|
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
|
|
|
|
Expect(env.ProbeMemory).ToNot(BeNil())
|
|
Expect(env.ServingFeaturePreference).To(Equal(system.ServingFeaturePreferenceTokens()))
|
|
})
|
|
})
|