From d32f45beca0e8abc7137fcf0ca9af7defda4b8b0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 21:59:23 +0000 Subject: [PATCH] fix(gallery): rank variants by engine name, not backend build tag Variant auto-selection ranked candidates with SystemState.BackendPreferenceTokens, but that function and the variant ranker speak different vocabularies. BackendPreferenceTokens returns BUILD TAGS ("cuda", "rocm", "sycl", "vulkan", "metal", "cpu"). It exists to match installed backend build directory names like "llama-cpp-cuda-12" during alias resolution in ListSystemBackends. Variant ranking instead matches a gallery entry's `backend:` value, which is an ENGINE NAME: "llama-cpp", "vllm", "vllm-omni", "sglang", "mlx" and the rest. No engine name in gallery/index.yaml contains "cuda", "rocm", "sycl" or "vulkan". preferenceRank matches by substring, so on an NVIDIA host the tokens [cuda, vulkan, cpu] matched neither "llama-cpp" nor "vllm", every candidate scored identically and size alone decided. The NVIDIA, AMD, Intel, darwin-x86 and vulkan rules were all inert. Only metal appeared to work, and only because the token "mlx" happens to equal an engine name. The mismatch does not error, it silently deletes the feature. Separate the two vocabularies. backendBuildTagPreferenceRules keeps the build tags and its original output for every capability, including metal, whose "mlx" token is removed again; its alias-resolution consumer is byte-identical to before. engineNamePreferenceRules is new, holds engine names, and is read by the new EnginePreferenceTokens, which HostResolveEnv wires into the renamed ResolveEnv.EnginePreference. Both tables sit adjacent under one block comment naming each vocabulary and each consumer, and share one lookup helper so their semantics cannot drift. On NVIDIA the order is vLLM, then SGLang, then llama-cpp: vLLM is the throughput engine and a model published with a vLLM build is published that way because that build is the one worth running. AMD and Intel get the same order, since rocm and intel builds of both serving engines ship. Metal prefers mlx over llama-cpp. Vulkan prefers llama-cpp, the only LLM engine with a Vulkan build. darwin-x86 and unknown capabilities are deliberately absent rather than guessed at, degrading to the size-only ordering that predates preference. preferenceRank stays generic and names no engine and no capability, so adding a runtime remains a one-line table edit. Specs pin the NVIDIA and metal rules through the live table and the real HostResolveEnv wiring, so emptying the engine table or wiring the build tag source back in both go red. A regression table asserts BackendPreferenceTokens' original output per capability, and mirrored locks assert neither table carries the other's vocabulary. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/describe_variants_test.go | 26 ++- core/gallery/models.go | 13 +- core/gallery/resolve_variant.go | 32 +-- core/gallery/resolve_variant_test.go | 262 ++++++++++++++++++++----- pkg/system/capabilities.go | 139 ++++++++++--- pkg/system/capabilities_test.go | 113 +++++++++-- 6 files changed, 470 insertions(+), 115 deletions(-) diff --git a/core/gallery/describe_variants_test.go b/core/gallery/describe_variants_test.go index 28c20091c..4c20aca77 100644 --- a/core/gallery/describe_variants_test.go +++ b/core/gallery/describe_variants_test.go @@ -224,8 +224,8 @@ var _ = Describe("DescribeVariants", func() { Expect(view.AutoSelected).To(Equal("qwen3-8b-vllm-awq")) }) - It("matches what installing would pick when the host prefers a runtime", func() { - // The picker and the installer both have to apply backend + It("matches what installing would pick when the host prefers an engine", func() { + // The picker and the installer both have to apply engine // preference, or a Mac would be shown the GGUF build and handed the // MLX one. Asserting the agreement AND the name pins both halves. mlx := newModel("qwen3-8b-mlx-4bit", "mlx") @@ -237,7 +237,9 @@ var _ = Describe("DescribeVariants", func() { "qwen3-8b-gguf-q8": gib(9), "qwen3-8b-mlx-4bit": gib(5), }) - env.BackendPreference = []string{"mlx", "metal", "cpu"} + // Engine names as SystemState.EnginePreferenceTokens reports them for + // metal. Build tags would match no gallery `backend:` value here. + env.EnginePreference = []string{"mlx", "llama-cpp"} agreesWithInstall(env) @@ -246,6 +248,24 @@ var _ = Describe("DescribeVariants", func() { Expect(view.AutoSelected).To(Equal("qwen3-8b-mlx-4bit")) }) + It("matches what installing would pick when the host prefers vLLM", func() { + // The NVIDIA rule the user asked for, checked through the listing so + // the picker and the installer cannot drift on it. The GGUF build is + // deliberately the LARGER one, so only preference can produce this + // answer: size alone would name the llama.cpp build. + env := probing(gib(64), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(9), + "qwen3-8b-gguf-q8": gib(20), + }) + env.EnginePreference = []string{"vllm", "sglang", "llama-cpp"} + + agreesWithInstall(env) + + view, err := gallery.DescribeVariants(models, base, env) + Expect(err).ToNot(HaveOccurred()) + Expect(view.AutoSelected).To(Equal("qwen3-8b-vllm-awq")) + }) + It("names the entry itself when no variant fits", func() { view, err := gallery.DescribeVariants(models, base, probing(gib(2), map[string]uint64{ "qwen3-8b-vllm-awq": gib(20), diff --git a/core/gallery/models.go b/core/gallery/models.go index aa5c91cce..0601f0741 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -275,11 +275,14 @@ func HostResolveEnv(ctx context.Context, systemState *system.SystemState) Resolv BackendCompatible: func(backend string) bool { return systemState.IsBackendCompatible(backend, "") }, - // The ranking half of the hardware story, from the same table that - // already picks among installed backend builds. IsBackendCompatible - // only rules out what cannot run; on a Mac both an MLX and a llama.cpp - // build can, and this is what makes the native runtime win. - BackendPreference: systemState.BackendPreferenceTokens(), + // The ranking half of the hardware story. IsBackendCompatible only rules + // out what cannot run; on a Mac both an MLX and a llama.cpp build can, + // and this is what makes the native runtime win. + // + // EnginePreferenceTokens, NOT BackendPreferenceTokens: a variant is + // matched on its gallery `backend:` engine name, and the latter reports + // build tags that no engine name contains. + EnginePreference: systemState.EnginePreferenceTokens(), ProbeMemory: func(target *GalleryModel) uint64 { return probeEntryMemory(ctx, target) }, diff --git a/core/gallery/resolve_variant.go b/core/gallery/resolve_variant.go index 6f9bf6a5c..9758dec63 100644 --- a/core/gallery/resolve_variant.go +++ b/core/gallery/resolve_variant.go @@ -66,10 +66,18 @@ type ResolveEnv struct { // A nil func treats every backend as runnable, the right default for a // caller with no view of the hardware. BackendCompatible func(backend string) bool - // BackendPreference lists the runtime tokens this host prefers, best first, - // as SystemState.BackendPreferenceTokens reports them (e.g. metal gives - // ["mlx", "metal", "cpu"], NVIDIA gives ["cuda", "vulkan", "cpu"]). A token - // is matched as a substring of a variant's backend name. + // EnginePreference lists the ENGINE NAMES this host prefers, best first, as + // SystemState.EnginePreferenceTokens reports them (e.g. NVIDIA gives + // ["vllm", "sglang", "llama-cpp"], metal gives ["mlx", "llama-cpp"]). A + // token is matched as a substring of a variant's backend name. + // + // The vocabulary is load bearing. VariantOption.Backend is a gallery entry's + // `backend:` value, which is an engine name and never carries a build tag, + // so build tags like "cuda" or "rocm" match NOTHING here. Do not wire + // SystemState.BackendPreferenceTokens into this field: that reports build + // tags for installed-build alias resolution, and the mismatch does not + // error, it scores every candidate equally and silently reduces selection to + // size alone. pkg/system/capabilities.go documents both tables. // // BackendCompatible answers "can this run here at all" and is a filter. // This answers "which of the things that CAN run here should win" and is @@ -82,7 +90,7 @@ type ResolveEnv struct { // ordering by size alone. That is the right default for a caller with no // view of the hardware, and it is what every host looked like before // preference existed. - BackendPreference []string + EnginePreference []string // ProbeMemory measures how much memory a referenced gallery entry needs, // without downloading it. A zero result means "could not tell", never // "needs nothing". @@ -103,28 +111,28 @@ func (e ResolveEnv) backendRuns(backend string) bool { return e.BackendCompatible(backend) } -// preferenceRank scores a backend against this host's preferred runtime order, +// preferenceRank scores a backend against this host's preferred engine order, // lower being better. // -// It walks BackendPreference generically and knows nothing about any particular -// backend or capability: everything a new runtime needs is expressed by the -// token table in pkg/system, so adding one never reaches this function. +// It walks EnginePreference generically and names no engine and no capability: +// everything a new runtime needs is expressed by the table in pkg/system, so +// adding one never reaches this function. // // A backend matching no token scores just below the least preferred known one. // It is never dropped, and a field where nothing matches scores uniformly, so // an unrecognised backend, an unrecognised capability and an absent preference // list all degrade to the same predictable place: ordering by size alone. func (e ResolveEnv) preferenceRank(backend string) int { - if len(e.BackendPreference) == 0 { + if len(e.EnginePreference) == 0 { return 0 } name := strings.ToLower(backend) - for i, token := range e.BackendPreference { + for i, token := range e.EnginePreference { if token != "" && strings.Contains(name, token) { return i } } - return len(e.BackendPreference) + return len(e.EnginePreference) } // VariantSelection is the outcome of a selection pass. diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index e4f72c77e..3ac9bec85 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -1,10 +1,14 @@ package gallery_test import ( + "context" + "os" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" ) var _ = Describe("VariantOption.EffectiveMemory", func() { @@ -243,18 +247,77 @@ var _ = Describe("SelectVariant", func() { }) }) - Describe("ranking by host backend preference", func() { - // The token lists below are exactly what - // SystemState.BackendPreferenceTokens reports for these hosts. They are - // spelled out rather than read from the live machine so the specs pin - // the intended behaviour on every CI runner. - darwinMetal := []string{"mlx", "metal", "cpu"} - nvidia := []string{"cuda", "vulkan", "cpu"} + Describe("ranking by host engine preference", func() { + // These are ENGINE NAMES, exactly what SystemState.EnginePreferenceTokens + // reports for these hosts and exactly what a gallery entry's `backend:` + // field holds. Build tags ("cuda", "rocm", "metal") belong to + // BackendPreferenceTokens and would match no engine name here, which is + // why every backend below is spelled as a real gallery engine. + // + // They are spelled out rather than read from the live machine so the + // specs pin the intended behaviour on every CI runner. + darwinMetal := []string{"mlx", "llama-cpp"} + nvidia := []string{"vllm", "sglang", "llama-cpp"} // A Mac runs both engines, so nothing is filtered here and preference is // the only thing that can decide. darwinRunsEverything := func(string) bool { return true } + It("prefers a vLLM build to a larger llama.cpp build on nvidia", func() { + // The rule the engine table exists to express, and the one that was + // silently inert while the ranker was fed build tags: no gallery + // engine name contains "cuda", so every candidate scored equal and + // the larger llama.cpp build won. Emptying the nvidia rule in + // pkg/system fails this spec. + options := []gallery.VariantOption{ + option("m-vllm-awq", "vllm", gib(8)), + option("m-gguf-q8", "llama-cpp", gib(24)), + base("m-gguf-q4", gib(4)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(64), + EnginePreference: nvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-vllm-awq")) + }) + + It("takes the larger llama.cpp build on the same nvidia host once preference is unknown", func() { + // The mirror of the spec above, proving the vLLM win comes from the + // preference list and not from anything intrinsic to the option set. + // This is the state the whole feature was stuck in before the two + // vocabularies were separated. + options := []gallery.VariantOption{ + option("m-vllm-awq", "vllm", gib(8)), + option("m-gguf-q8", "llama-cpp", gib(24)), + base("m-gguf-q4", gib(4)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(64), + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) + }) + + It("ranks a vllm-omni build with vllm, since the token is a substring", func() { + // Substring matching is deliberate: vllm-omni is a vLLM build and + // must inherit vLLM's rank rather than fall through to unranked. + options := []gallery.VariantOption{ + option("m-omni", "vllm-omni", gib(8)), + option("m-gguf-q8", "llama-cpp", gib(24)), + base("m-base", gib(4)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(64), + EnginePreference: nvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-omni")) + }) + It("prefers an MLX build to a larger llama.cpp build on darwin", func() { options := []gallery.VariantOption{ option("m-mlx-4bit", "mlx", gib(8)), @@ -265,13 +328,13 @@ var _ = Describe("SelectVariant", func() { selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ AvailableMemory: gib(64), BackendCompatible: darwinRunsEverything, - BackendPreference: darwinMetal, + EnginePreference: darwinMetal, }, "") Expect(err).ToNot(HaveOccurred()) Expect(selection.Option.Variant.Model).To(Equal("m-mlx-4bit")) }) - It("takes the larger llama.cpp build on the same host once preference is unknown", func() { + It("takes the larger llama.cpp build on the same darwin host once preference is unknown", func() { // The mirror of the spec above, proving the MLX win comes from the // preference list and not from anything intrinsic to the option set. options := []gallery.VariantOption{ @@ -288,75 +351,96 @@ var _ = Describe("SelectVariant", func() { Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) }) - It("prefers a CUDA build to a larger CPU-only build on nvidia", func() { - options := []gallery.VariantOption{ - option("m-cuda-q4", "cuda12-llama-cpp", gib(8)), - option("m-cpu-q8", "cpu-llama-cpp", gib(24)), - base("m-base", gib(4)), - } - - selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ - AvailableMemory: gib(64), - BackendPreference: nvidia, - }, "") - Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-cuda-q4")) - }) - - It("still picks the largest fitting build among equally preferred backends", func() { - // Preference must not flatten size ordering: with one runtime in + It("still picks the largest fitting build among equally preferred engines", func() { + // Preference must not flatten size ordering: with one engine in // play there is nothing left for it to decide. options := []gallery.VariantOption{ - option("m-cuda-q4", "cuda12-llama-cpp", gib(6)), - option("m-cuda-q8", "cuda12-llama-cpp", gib(12)), - option("m-cuda-f16", "cuda12-llama-cpp", gib(48)), + option("m-gguf-q4", "llama-cpp", gib(6)), + option("m-gguf-q8", "llama-cpp", gib(12)), + option("m-gguf-f16", "llama-cpp", gib(48)), base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ - AvailableMemory: gib(16), - BackendPreference: nvidia, + AvailableMemory: gib(16), + EnginePreference: nvidia, }, "") Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-cuda-q8")) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) }) - It("does not let a preferred backend rescue a build that does not fit", func() { + It("does not let a preferred engine rescue a build that does not fit", func() { // Fit is a filter and preference is only a ranking among survivors. - // The CUDA build is both preferred and too large, so the CPU build - // the host can actually hold has to win. + // The vLLM build is both preferred and too large, so the llama.cpp + // build the host can actually hold has to win. options := []gallery.VariantOption{ - option("m-cuda-f16", "cuda12-llama-cpp", gib(48)), - option("m-cpu-q4", "cpu-llama-cpp", gib(6)), + option("m-vllm-fp16", "vllm", gib(48)), + option("m-gguf-q4", "llama-cpp", gib(6)), base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ - AvailableMemory: gib(16), - BackendPreference: nvidia, + AvailableMemory: gib(16), + EnginePreference: nvidia, }, "") Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-cpu-q4")) - Expect(selection.Reasons).To(ContainElement(ContainSubstring("m-cuda-f16"))) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q4")) + Expect(selection.Reasons).To(ContainElement(ContainSubstring("m-vllm-fp16"))) }) - It("orders unrecognised backends by size rather than dropping them", func() { - // Neither engine name carries an nvidia token. Nothing is discarded + It("orders engines absent from the table by size rather than dropping them", func() { + // Neither engine appears in the nvidia rule. Nothing is discarded // and nothing is arbitrarily favoured; the host falls back to the // size-only behaviour it had before preference existed. + // + // The base is left unsized so it ranks in the tier below a proven + // fit. It is a llama.cpp build, which the nvidia rule DOES rank, and + // letting it compete in the same tier would prove preference works + // rather than proving unlisted engines degrade to size. options := []gallery.VariantOption{ - option("m-vllm-awq", "vllm", gib(12)), - option("m-sglang-fp8", "sglang", gib(6)), + option("m-diffusers", "diffusers", gib(12)), + option("m-transformers", "transformers", gib(6)), + base("m-base", 0), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(16), + EnginePreference: nvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-diffusers")) + Expect(selection.Reasons).To(BeEmpty()) + }) + + It("ranks sglang between vllm and llama-cpp on nvidia", func() { + // A judgement call worth pinning: sglang is a GPU serving engine of + // the same class as vllm, so it outranks the portable engine, but it + // sits behind vllm. + options := []gallery.VariantOption{ + option("m-vllm", "vllm", gib(4)), + option("m-sglang", "sglang", gib(6)), + option("m-gguf", "llama-cpp", gib(8)), base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ - AvailableMemory: gib(16), - BackendPreference: nvidia, + AvailableMemory: gib(16), + EnginePreference: nvidia, }, "") Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-vllm-awq")) - Expect(selection.Reasons).To(BeEmpty()) + Expect(selection.Option.Variant.Model).To(Equal("m-vllm")) + + withoutVLLM := []gallery.VariantOption{ + option("m-sglang", "sglang", gib(6)), + option("m-gguf", "llama-cpp", gib(8)), + base("m-base", gib(2)), + } + selection, err = gallery.SelectVariant(withoutVLLM, gallery.ResolveEnv{ + AvailableMemory: gib(16), + EnginePreference: nvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-sglang")) }) It("still installs the base when nothing fits, whatever the host prefers", func() { @@ -369,7 +453,7 @@ var _ = Describe("SelectVariant", func() { selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ AvailableMemory: gib(4), BackendCompatible: darwinRunsEverything, - BackendPreference: darwinMetal, + EnginePreference: darwinMetal, }, "") Expect(err).ToNot(HaveOccurred()) Expect(selection.Option.Variant.Model).To(Equal("m-base")) @@ -389,7 +473,7 @@ var _ = Describe("SelectVariant", func() { selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ AvailableMemory: gib(64), BackendCompatible: darwinRunsEverything, - BackendPreference: darwinMetal, + EnginePreference: darwinMetal, }, "m-gguf-q8") Expect(err).ToNot(HaveOccurred()) Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) @@ -502,3 +586,79 @@ var _ = Describe("SelectVariant", func() { }) }) }) + +var _ = Describe("HostResolveEnv engine preference wiring", func() { + // The specs above feed SelectVariant a hand-written token list, which proves + // the ranker but not that the host actually reaches the engine table. These + // drive the REAL table through the REAL wiring, so emptying + // engineNamePreferenceRules in pkg/system, or reverting this field to + // BackendPreferenceTokens, fails here. + var origEnv string + const capabilityEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY" + + BeforeEach(func() { + origEnv = os.Getenv(capabilityEnv) + }) + + AfterEach(func() { + if origEnv != "" { + os.Setenv(capabilityEnv, origEnv) + } else { + os.Unsetenv(capabilityEnv) + } + }) + + envFor := func(capability string) gallery.ResolveEnv { + GinkgoHelper() + Expect(os.Setenv(capabilityEnv, capability)).To(Succeed()) + return gallery.HostResolveEnv(context.Background(), &system.SystemState{}) + } + + It("hands the ranker engine names an NVIDIA host's gallery entries can match", func() { + preference := envFor("nvidia-cuda-12").EnginePreference + Expect(preference).To(Equal([]string{"vllm", "sglang", "llama-cpp"})) + }) + + It("installs the vLLM build over a larger llama.cpp one on a real NVIDIA host", func() { + // End to end on the live table: the exact behaviour that was silently + // dead while the ranker was fed build tags. + env := envFor("nvidia-cuda-12") + env.AvailableMemory = 64 * 1024 * 1024 * 1024 + + options := []gallery.VariantOption{ + {Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 8 * 1024 * 1024 * 1024}, + {Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 24 * 1024 * 1024 * 1024}, + } + + selection, err := gallery.SelectVariant(options, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-vllm")) + }) + + It("installs the MLX build over a larger llama.cpp one on a real darwin host", func() { + env := envFor("metal") + env.AvailableMemory = 64 * 1024 * 1024 * 1024 + // The real gate rejects mlx off darwin, and this spec is about ranking. + env.BackendCompatible = func(string) bool { return true } + + options := []gallery.VariantOption{ + {Variant: gallery.Variant{Model: "m-mlx"}, Backend: "mlx", ProbedMemory: 8 * 1024 * 1024 * 1024}, + {Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 24 * 1024 * 1024 * 1024}, + } + + selection, err := gallery.SelectVariant(options, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-mlx")) + }) + + It("carries no build tag into the ranker, whatever the host", func() { + // The wiring-level lock. BackendPreferenceTokens returns build tags for + // every capability, so if it is ever wired back into this field, one of + // these tags shows up here. + for _, capability := range []string{"nvidia-cuda-12", "amd", "intel", "metal", "vulkan", "default"} { + Expect(envFor(capability).EnginePreference).ToNot( + ContainElements("cuda", "rocm", "hip", "sycl", "metal", "cpu", "darwin-x86"), + "capability %q leaked a build tag into the variant ranker", capability) + } + }) +}) diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 6aaf85105..172283248 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -45,19 +45,47 @@ const ( backendTokenHIP = "hip" backendTokenSYCL = "sycl" backendTokenCPU = "cpu" + + // Engine names (private). Unlike the tokens above these are whole backend + // identities as a gallery entry's `backend:` field spells them, not build + // tags. See the two preference tables below for why the distinction matters. + engineVLLM = "vllm" + engineSGLang = "sglang" + engineLlamaCpp = "llama-cpp" + engineMLX = "mlx" ) -// backendPreferenceRule maps a detected capability to the runtime tokens that -// capability prefers, best first. +// There are TWO preference tables below and they speak DIFFERENT VOCABULARIES. +// Merging them looks tempting and silently breaks one of the two consumers, +// because a token that means something in one vocabulary means nothing in the +// other. Read this before editing either. // -// This table is the single source of truth for "what should this host run -// first", consumed both by concrete-backend alias resolution and by gallery -// variant auto-selection. Keeping it declarative is deliberate: LocalAI gains -// backends and hardware targets continuously, and expressing a new preference -// has to stay a one-line edit here rather than a change to any ranking code. +// - backendBuildTagPreferenceRules holds BUILD TAGS ("cuda", "rocm", "metal"). +// They are matched against INSTALLED BACKEND BUILD DIRECTORY NAMES such as +// "llama-cpp-cuda-12" or "cuda12-vllm". Consumer: alias resolution in +// ListSystemBackends (core/gallery/backends.go), which picks which installed +// build of one alias to run. // -// To add a capability, append a rule. To reorder one host's runtimes, reorder -// its tokens. Nothing else needs to change. +// - engineNamePreferenceRules holds ENGINE NAMES ("vllm", "llama-cpp", "mlx"). +// They are matched against a gallery entry's `backend:` value, which never +// carries a build tag: no entry in gallery/index.yaml contains "cuda", +// "rocm", "sycl" or "vulkan" anywhere in its backend name. Consumer: gallery +// variant auto-selection (core/gallery/resolve_variant.go), which picks +// which build of one model's weights to install. +// +// Feeding build tags to the variant ranker matches nothing, which does not +// error: every candidate simply scores equal and size alone decides, so the +// preference silently stops existing. That is exactly the bug this split fixes. + +// backendPreferenceRule maps a detected capability to preferred tokens, best +// first. Both tables share this shape; only their vocabulary differs. +type backendPreferenceRule struct { + capabilityPrefix string + tokens []string +} + +// backendBuildTagPreferenceRules is the BUILD TAG table. See the block comment +// above for the vocabulary contract. // // Matching is by capability PREFIX, because a detected capability is refined at // runtime ("nvidia" becomes "nvidia-cuda-12" when the toolkit is present) and a @@ -65,31 +93,62 @@ const ( // in order, so a more specific prefix must precede any rule it shares a prefix // with. // -// Tokens are matched as substrings of a backend name, so "cuda" covers -// "cuda12-llama-cpp" and "mlx" covers "metal-mlx" alike. A backend matching no -// token is not an error and is never discarded; it simply sorts below every -// recognised one. -type backendPreferenceRule struct { - capabilityPrefix string - tokens []string -} - -var backendPreferenceRules = []backendPreferenceRule{ +// Tokens are matched as substrings of a build directory name, so "cuda" covers +// "cuda12-llama-cpp". A build matching no token is not an error and is never +// discarded; it simply sorts below every recognised one. +var backendBuildTagPreferenceRules = []backendPreferenceRule{ {Nvidia, []string{backendTokenCUDA, vulkan, backendTokenCPU}}, {AMD, []string{backendTokenROCM, backendTokenHIP, vulkan, backendTokenCPU}}, {Intel, []string{backendTokenSYCL, Intel, backendTokenCPU}}, - // MLX outranks metal because on Apple silicon it is the native accelerated - // runtime, whereas a metal-enabled GGUF build is the portable engine merely - // compiled with GPU offload. - {metal, []string{backendTokenMLX, backendTokenMetal, backendTokenCPU}}, + {metal, []string{backendTokenMetal, backendTokenCPU}}, {darwinX86, []string{darwinX86, backendTokenCPU}}, {vulkan, []string{vulkan, backendTokenCPU}}, } -// defaultBackendPreferenceTokens is what a host with no matching rule prefers. +// defaultBackendBuildTagTokens is what a host with no matching rule prefers. // A capability nobody has taught this table about degrades to plain CPU rather // than to an error or to an empty list. -var defaultBackendPreferenceTokens = []string{backendTokenCPU} +var defaultBackendBuildTagTokens = []string{backendTokenCPU} + +// engineNamePreferenceRules is the ENGINE NAME table. See the block comment +// above for the vocabulary contract. +// +// Capability matching is by prefix, exactly as the build tag table does it. +// +// Tokens are matched as SUBSTRINGS of an engine name, which is load bearing: +// "vllm" also covers "vllm-omni", "mlx" also covers "mlx-vlm" and "mlx-audio", +// and "llama-cpp" also covers "ik-llama-cpp". Each of those is a build of the +// engine named by the token, so ranking them together is correct. Order the +// tokens so no token is a substring of an engine that should rank differently. +// +// A capability is deliberately ABSENT rather than guessed at when no engine +// ordering can be justified for it. An absent rule degrades to ordering by size +// alone, which is the behaviour that predates preference and is always safe. +// darwin-x86 is absent for that reason: nothing accelerates there, so no engine +// deserves to outrank another. +var engineNamePreferenceRules = []backendPreferenceRule{ + // vLLM first on every host with a dedicated serving engine build. It is the + // throughput engine, and a model published with a vLLM build is published + // that way precisely because that build is the one worth running. + // SGLang sits directly behind it: same class of GPU serving engine, ships + // cuda/rocm/intel builds alike, but it is behind vLLM because vLLM covers + // far more of the gallery. llama-cpp is the portable fallback. + {Nvidia, []string{engineVLLM, engineSGLang, engineLlamaCpp}}, + {AMD, []string{engineVLLM, engineSGLang, engineLlamaCpp}}, + {Intel, []string{engineVLLM, engineSGLang, engineLlamaCpp}}, + // MLX is the native accelerated runtime on Apple silicon, whereas a + // metal-enabled GGUF build is the portable engine merely compiled with GPU + // offload. No vLLM or SGLang build targets metal, so neither is listed. + {metal, []string{engineMLX, engineLlamaCpp}}, + // A Vulkan host has exactly one LLM engine with a Vulkan build, so a + // llama-cpp variant is the only one that will use the GPU at all. + {vulkan, []string{engineLlamaCpp}}, +} + +// defaultEnginePreferenceTokens is empty on purpose. A host with no detected +// accelerator has no reason to prefer one engine over another, and an empty +// list is what the ranker reads as "order by size alone". +var defaultEnginePreferenceTokens = []string{} var ( cuda13DirExists bool @@ -232,18 +291,40 @@ func (s *SystemState) getSystemCapabilities() string { // backend implementation order for the current system capability. Callers can use // these tokens to select the most appropriate concrete backend among multiple // candidates sharing the same alias (e.g., "llama-cpp"). -// The rules live in backendPreferenceRules above; this function only looks them -// up, so teaching LocalAI about a new runtime never means editing logic. +// +// These are BUILD TAGS matched against installed build directory names. For +// engine names as a gallery entry spells them, use EnginePreferenceTokens. func (s *SystemState) BackendPreferenceTokens() []string { + return s.preferenceTokens(backendBuildTagPreferenceRules, defaultBackendBuildTagTokens) +} + +// EnginePreferenceTokens returns the engine names this host prefers, best first, +// for ranking gallery model variants against each other. +// +// These are ENGINE NAMES matched against a gallery entry's `backend:` value +// ("vllm", "llama-cpp", "mlx"), never build tags. Feeding this function's output +// to installed-build alias resolution, or BackendPreferenceTokens' output to +// variant ranking, matches nothing and silently disables the preference. +// +// An empty result is normal and means "no engine ordering applies here", which +// the ranker reads as ordering by size alone. +func (s *SystemState) EnginePreferenceTokens() []string { + return s.preferenceTokens(engineNamePreferenceRules, defaultEnginePreferenceTokens) +} + +// preferenceTokens resolves the current capability against one preference table. +// The rules live in the tables above; this only looks them up, so teaching +// LocalAI about a new runtime never means editing logic. +func (s *SystemState) preferenceTokens(rules []backendPreferenceRule, fallback []string) []string { capStr := strings.ToLower(s.getSystemCapabilities()) - for _, rule := range backendPreferenceRules { + for _, rule := range rules { if strings.HasPrefix(capStr, rule.capabilityPrefix) { // Copied so a caller cannot mutate the shared table out from under // every other host lookup. return slices.Clone(rule.tokens) } } - return slices.Clone(defaultBackendPreferenceTokens) + return slices.Clone(fallback) } // DetectedCapability returns the raw detected capability string (e.g. "metal", diff --git a/pkg/system/capabilities_test.go b/pkg/system/capabilities_test.go index eb3c2e70c..9ac30b991 100644 --- a/pkg/system/capabilities_test.go +++ b/pkg/system/capabilities_test.go @@ -149,22 +149,36 @@ var _ = Describe("BackendPreferenceTokens", func() { return (&SystemState{}).BackendPreferenceTokens() } - It("ranks MLX above metal on apple silicon", func() { - // MLX is the native accelerated runtime there, so it must outrank a - // metal-enabled build of the portable engine. - Expect(tokensFor(metal)).To(Equal([]string{"mlx", "metal", "cpu"})) - }) + // This table is a REGRESSION LOCK, not a description. These are build tags + // matched against installed backend build directory names, and the only + // consumer is alias resolution in ListSystemBackends. Every value below is + // the output this function had before engine preference existed. + // + // Engine names ("vllm", "mlx", "llama-cpp") belong to EnginePreferenceTokens + // and MUST NOT appear here. Merging the two vocabularies is exactly the + // mistake this table exists to catch: it does not error, it silently makes + // one of the two consumers rank everything equally. + DescribeTable("returns the original build tags for every capability", + func(capability string, want []string) { + Expect(tokensFor(capability)).To(Equal(want)) + }, + Entry("nvidia", "nvidia-cuda-12", []string{"cuda", "vulkan", "cpu"}), + Entry("amd", AMD, []string{"rocm", "hip", "vulkan", "cpu"}), + Entry("intel", Intel, []string{"sycl", "intel", "cpu"}), + Entry("metal", metal, []string{"metal", "cpu"}), + Entry("darwin-x86", darwinX86, []string{"darwin-x86", "cpu"}), + Entry("vulkan", vulkan, []string{"vulkan", "cpu"}), + Entry("unknown capability", "some-future-accelerator", []string{"cpu"}), + ) - It("keeps the vendor order every other capability already had", func() { - Expect(tokensFor("nvidia-cuda-12")).To(Equal([]string{"cuda", "vulkan", "cpu"})) - Expect(tokensFor(AMD)).To(Equal([]string{"rocm", "hip", "vulkan", "cpu"})) - Expect(tokensFor(Intel)).To(Equal([]string{"sycl", "intel", "cpu"})) - Expect(tokensFor(darwinX86)).To(Equal([]string{"darwin-x86", "cpu"})) - Expect(tokensFor(vulkan)).To(Equal([]string{"vulkan", "cpu"})) - }) - - It("degrades an unknown capability to cpu rather than to nothing", func() { - Expect(tokensFor("some-future-accelerator")).To(Equal([]string{"cpu"})) + It("carries no engine name in any capability's tokens", func() { + // The generic form of the lock above: whatever anyone adds to the build + // tag table later, an engine name in it is a merged vocabulary. + engineNames := []string{"vllm", "sglang", "llama-cpp", "mlx"} + for _, capability := range []string{"nvidia", AMD, Intel, metal, darwinX86, vulkan, "default"} { + Expect(tokensFor(capability)).ToNot(ContainElements(engineNames), + "capability %q leaked an engine name into the build tag table", capability) + } }) It("matches a capability by prefix so a refined one keeps its vendor order", func() { @@ -178,6 +192,75 @@ var _ = Describe("BackendPreferenceTokens", func() { }) }) +var _ = Describe("EnginePreferenceTokens", func() { + var origEnv string + + BeforeEach(func() { + origEnv = os.Getenv(capabilityEnv) + }) + + AfterEach(func() { + if origEnv != "" { + os.Setenv(capabilityEnv, origEnv) + } else { + os.Unsetenv(capabilityEnv) + } + }) + + tokensFor := func(capability string) []string { + GinkgoHelper() + Expect(os.Setenv(capabilityEnv, capability)).To(Succeed()) + return (&SystemState{}).EnginePreferenceTokens() + } + + It("puts vLLM ahead of llama.cpp on nvidia", func() { + // The rule the whole engine table exists to express. A build tag list + // here would match no engine name at all and this would read empty. + Expect(tokensFor("nvidia-cuda-12")).To(Equal([]string{"vllm", "sglang", "llama-cpp"})) + }) + + It("puts MLX ahead of llama.cpp on apple silicon", func() { + Expect(tokensFor(metal)).To(Equal([]string{"mlx", "llama-cpp"})) + }) + + It("gives the GPU serving engines to every vendor that has builds of them", func() { + Expect(tokensFor(AMD)).To(Equal([]string{"vllm", "sglang", "llama-cpp"})) + Expect(tokensFor(Intel)).To(Equal([]string{"vllm", "sglang", "llama-cpp"})) + }) + + It("prefers llama.cpp on vulkan, the only engine with a vulkan build", func() { + Expect(tokensFor(vulkan)).To(Equal([]string{"llama-cpp"})) + }) + + It("names only engines, never a build tag", func() { + // The mirror of the lock on BackendPreferenceTokens. A build tag here + // matches no gallery `backend:` value and silently disables ranking. + buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86"} + for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan} { + Expect(tokensFor(capability)).ToNot(ContainElements(buildTags), + "capability %q leaked a build tag into the engine table", capability) + } + }) + + It("leaves a capability with no justified engine order empty rather than guessing", func() { + // Empty is read by the variant ranker as "order by size alone", which is + // the behaviour that predates preference and is always safe. + Expect(tokensFor(darwinX86)).To(BeEmpty()) + Expect(tokensFor("default")).To(BeEmpty()) + Expect(tokensFor("some-future-accelerator")).To(BeEmpty()) + }) + + It("matches a capability by prefix so a refined one keeps its engine order", func() { + Expect(tokensFor("nvidia-l4t-cuda-13")).To(Equal(tokensFor("nvidia"))) + }) + + It("hands out a copy so one caller cannot corrupt another's lookup", func() { + first := tokensFor("nvidia") + first[0] = "clobbered" + Expect(tokensFor("nvidia")[0]).To(Equal("vllm")) + }) +}) + var _ = Describe("CapabilityFilterDisabled", func() { var origEnv string