diff --git a/.agents/adding-backends.md b/.agents/adding-backends.md index 270fe7a02..e02635a69 100644 --- a/.agents/adding-backends.md +++ b/.agents/adding-backends.md @@ -244,6 +244,18 @@ Leaving your backend out is a valid choice when no ordering can be justified for it. It then ranks below every known engine and selection falls back to size, which is the behaviour that predates preference. +**Leaving a whole capability out is not.** A missing row gives that host an +empty preference list, so size alone decides among everything that survives the +filters, and the filter will not save you: `IsBackendCompatible` derives hardware +support from the engine NAME, so `vllm` and `sglang` carry no darwin, cuda, rocm +or sycl token and are never dropped on a host with no GPU. That is why `default` +(no usable accelerator, including a GPU under the 4 GiB VRAM floor) and +`darwin-x86` both have rows putting `llama-cpp` first. Every capability +`getSystemCapabilities()` can return needs a row unless every engine really is +equally at home there. When you add one, enumerate the engines you are demoting +rather than relying on them falling through unmatched: unmatched engines all tie +with each other, so size decides among them. + ## Documenting the backend (README + docs) A backend is not "added" until it is discoverable. Update the user-facing docs: diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index 3ac9bec85..671d8789d 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -3,6 +3,8 @@ package gallery_test import ( "context" "os" + "path/filepath" + "runtime" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -593,11 +595,15 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() { // drive the REAL table through the REAL wiring, so emptying // engineNamePreferenceRules in pkg/system, or reverting this field to // BackendPreferenceTokens, fails here. - var origEnv string + var origEnv, origRunFileEnv string const capabilityEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY" + const capabilityRunFileEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY_RUN_FILE" + // What getSystemCapabilities reports for a host with no usable accelerator. + const noGPUCapability = "default" BeforeEach(func() { origEnv = os.Getenv(capabilityEnv) + origRunFileEnv = os.Getenv(capabilityRunFileEnv) }) AfterEach(func() { @@ -606,6 +612,11 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() { } else { os.Unsetenv(capabilityEnv) } + if origRunFileEnv != "" { + os.Setenv(capabilityRunFileEnv, origRunFileEnv) + } else { + os.Unsetenv(capabilityRunFileEnv) + } }) envFor := func(capability string) gallery.ResolveEnv { @@ -651,6 +662,80 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() { Expect(selection.Option.Variant.Model).To(Equal("m-mlx")) }) + It("installs the llama.cpp build over a larger vLLM one on a host with no GPU", func() { + // The hole this rule closes. IsBackendCompatible keys on the engine + // name, and "vllm" carries no darwin/cuda/rocm/sycl token, so a vLLM + // build is NOT filtered out here. Emptying the default rule in + // pkg/system puts the larger vLLM build back on a CPU-only box. + env := envFor(noGPUCapability) + env.AvailableMemory = 64 * 1024 * 1024 * 1024 + + options := []gallery.VariantOption{ + {Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 24 * 1024 * 1024 * 1024}, + {Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 8 * 1024 * 1024 * 1024}, + } + + selection, err := gallery.SelectVariant(options, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf")) + }) + + It("installs the llama.cpp build over a larger vLLM one on an intel mac", func() { + env := envFor("darwin-x86") + env.AvailableMemory = 64 * 1024 * 1024 * 1024 + + options := []gallery.VariantOption{ + {Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 24 * 1024 * 1024 * 1024}, + {Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 8 * 1024 * 1024 * 1024}, + } + + selection, err := gallery.SelectVariant(options, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf")) + }) + + It("still installs a vLLM build on a host with no GPU when it is the only one offered", func() { + // Preference ORDERS survivors, it never filters them. Demoting vLLM must + // not make a model published only as a vLLM build uninstallable. + env := envFor(noGPUCapability) + env.AvailableMemory = 64 * 1024 * 1024 * 1024 + + options := []gallery.VariantOption{ + {Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 8 * 1024 * 1024 * 1024}, + } + + selection, err := gallery.SelectVariant(options, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-vllm")) + }) + + It("prefers llama.cpp on a GPU host with too little VRAM to serve from", func() { + // This host has a GPU, yet getSystemCapabilities reports "default" + // because it is under the 4 GiB floor. Driven through the real detector + // rather than a forced capability, so that mapping is exercised too. + if runtime.GOOS == "darwin" { + Skip("darwin reports metal or darwin-x86 before the VRAM floor is consulted") + } + Expect(os.Unsetenv(capabilityEnv)).To(Succeed()) + // A capability run file on the machine would override detection. + Expect(os.Setenv(capabilityRunFileEnv, filepath.Join(GinkgoT().TempDir(), "absent"))).To(Succeed()) + + state := &system.SystemState{GPUVendor: system.Nvidia, VRAM: 2 * 1024 * 1024 * 1024} + Expect(state.DetectedCapability()).To(Equal(noGPUCapability)) + + env := gallery.HostResolveEnv(context.Background(), state) + env.AvailableMemory = 64 * 1024 * 1024 * 1024 + + options := []gallery.VariantOption{ + {Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 24 * 1024 * 1024 * 1024}, + {Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 8 * 1024 * 1024 * 1024}, + } + + selection, err := gallery.SelectVariant(options, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf")) + }) + It("carries no build tag into the ranker, whatever the host", func() { // The wiring-level lock. BackendPreferenceTokens returns build tags for // every capability, so if it is ever wired back into this field, one of diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 172283248..5ceea9307 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -124,8 +124,14 @@ var defaultBackendBuildTagTokens = []string{backendTokenCPU} // A capability is deliberately ABSENT rather than guessed at when no engine // ordering can be justified for it. An absent rule degrades to ordering by size // alone, which is the behaviour that predates preference and is always safe. -// darwin-x86 is absent for that reason: nothing accelerates there, so no engine -// deserves to outrank another. +// +// Safe, however, only where every candidate engine is equally at home on the +// host. Where one is not, the rule has to be written here, because the hardware +// filter will not write it: IsBackendCompatible derives support from the engine +// NAME, and "vllm" and "sglang" contain none of the darwin, cuda, rocm or sycl +// tokens it keys on, so a GPU serving engine is never filtered out on a host +// with no GPU. Left unranked there, it wins on size alone whenever its build is +// the larger one, and a CPU-only box installs vLLM over llama.cpp. var engineNamePreferenceRules = []backendPreferenceRule{ // vLLM first on every host with a dedicated serving engine build. It is the // throughput engine, and a model published with a vLLM build is published @@ -143,11 +149,35 @@ var engineNamePreferenceRules = []backendPreferenceRule{ // A Vulkan host has exactly one LLM engine with a Vulkan build, so a // llama-cpp variant is the only one that will use the GPU at all. {vulkan, []string{engineLlamaCpp}}, + // No usable accelerator: either nothing was detected, or a GPU was found + // with too little VRAM to serve from, both of which report "default". + // llama.cpp is the engine built to run well on a CPU; vLLM and SGLang are + // GPU serving engines whose CPU paths exist but are not what an unattended + // auto-selection should hand a CPU-only box. + // + // The GPU engines are enumerated behind llama.cpp rather than left + // unmatched. Listing llama.cpp alone would already make it win, since an + // unmatched engine ranks below every listed one, but it would leave vLLM + // and SGLang tied with each other and with any engine nobody has ranked + // yet, so download size would decide among them. Naming them fixes that + // order and says the omission of a GPU engine from the top spot is a + // decision rather than an oversight. Ranking never filters, so a vLLM + // variant offered alone is still installed here. + {defaultCapability, []string{engineLlamaCpp, engineVLLM, engineSGLang}}, + // An Intel Mac has no accelerated runtime at all: MLX needs Apple silicon, + // and no vLLM or SGLang build targets darwin. Same ordering, same reasons. + // MLX is deliberately left unlisted so it ranks last, since + // IsBackendCompatible admits any darwin-tokened engine on this capability + // and will not drop it. + {darwinX86, []string{engineLlamaCpp, engineVLLM, engineSGLang}}, } -// defaultEnginePreferenceTokens is empty on purpose. A host with no detected -// accelerator has no reason to prefer one engine over another, and an empty -// list is what the ranker reads as "order by size alone". +// defaultEnginePreferenceTokens is empty on purpose. It is now only reached by +// a capability nobody has taught this table about, which an operator can force +// via LOCALAI_FORCE_META_BACKEND_CAPABILITY; the detected "default" has its own +// rule above. Guessing an order for hardware we know nothing about would be +// worse than ordering by size alone, which is what the ranker reads an empty +// list as and is the behaviour that predates preference. var defaultEnginePreferenceTokens = []string{} var ( diff --git a/pkg/system/capabilities_test.go b/pkg/system/capabilities_test.go index 9ac30b991..2e589327f 100644 --- a/pkg/system/capabilities_test.go +++ b/pkg/system/capabilities_test.go @@ -236,17 +236,37 @@ var _ = Describe("EnginePreferenceTokens", func() { // The mirror of the lock on BackendPreferenceTokens. A build tag here // matches no gallery `backend:` value and silently disables ranking. buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86"} - for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan} { + for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan, defaultCapability, darwinX86} { Expect(tokensFor(capability)).ToNot(ContainElements(buildTags), "capability %q leaked a build tag into the engine table", capability) } }) - It("leaves a capability with no justified engine order empty rather than guessing", func() { + It("prefers llama.cpp on a host with no usable accelerator", func() { + // Nothing filters a GPU serving engine out here: IsBackendCompatible + // keys on the engine name, and "vllm" carries no hardware token. Without + // this rule a larger vLLM build would win a CPU-only host on size. + Expect(tokensFor(defaultCapability)).To(Equal([]string{"llama-cpp", "vllm", "sglang"})) + }) + + It("prefers llama.cpp on an intel mac, where nothing accelerates", func() { + Expect(tokensFor(darwinX86)).To(Equal([]string{"llama-cpp", "vllm", "sglang"})) + }) + + It("ranks the GPU serving engines below llama.cpp rather than leaving them tied", func() { + // Enumerating them is what stops download size deciding between vLLM and + // SGLang once llama.cpp is out of the running. + for _, capability := range []string{defaultCapability, darwinX86} { + tokens := tokensFor(capability) + Expect(tokens).To(HaveLen(3), "capability %q", capability) + Expect(tokens[0]).To(Equal("llama-cpp"), "capability %q", capability) + } + }) + + It("leaves a capability nobody has taught the table about empty rather than guessing", func() { // Empty is read by the variant ranker as "order by size alone", which is - // the behaviour that predates preference and is always safe. - Expect(tokensFor(darwinX86)).To(BeEmpty()) - Expect(tokensFor("default")).To(BeEmpty()) + // the behaviour that predates preference and is always safe. Only a + // forced, unrecognised capability lands here now. Expect(tokensFor("some-future-accelerator")).To(BeEmpty()) })