diff --git a/core/gallery/describe_variants_test.go b/core/gallery/describe_variants_test.go index 28c20091c..4c20aca77 100644 --- a/core/gallery/describe_variants_test.go +++ b/core/gallery/describe_variants_test.go @@ -224,8 +224,8 @@ var _ = Describe("DescribeVariants", func() { Expect(view.AutoSelected).To(Equal("qwen3-8b-vllm-awq")) }) - It("matches what installing would pick when the host prefers a runtime", func() { - // The picker and the installer both have to apply backend + It("matches what installing would pick when the host prefers an engine", func() { + // The picker and the installer both have to apply engine // preference, or a Mac would be shown the GGUF build and handed the // MLX one. Asserting the agreement AND the name pins both halves. mlx := newModel("qwen3-8b-mlx-4bit", "mlx") @@ -237,7 +237,9 @@ var _ = Describe("DescribeVariants", func() { "qwen3-8b-gguf-q8": gib(9), "qwen3-8b-mlx-4bit": gib(5), }) - env.BackendPreference = []string{"mlx", "metal", "cpu"} + // Engine names as SystemState.EnginePreferenceTokens reports them for + // metal. Build tags would match no gallery `backend:` value here. + env.EnginePreference = []string{"mlx", "llama-cpp"} agreesWithInstall(env) @@ -246,6 +248,24 @@ var _ = Describe("DescribeVariants", func() { Expect(view.AutoSelected).To(Equal("qwen3-8b-mlx-4bit")) }) + It("matches what installing would pick when the host prefers vLLM", func() { + // The NVIDIA rule the user asked for, checked through the listing so + // the picker and the installer cannot drift on it. The GGUF build is + // deliberately the LARGER one, so only preference can produce this + // answer: size alone would name the llama.cpp build. + env := probing(gib(64), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(9), + "qwen3-8b-gguf-q8": gib(20), + }) + env.EnginePreference = []string{"vllm", "sglang", "llama-cpp"} + + agreesWithInstall(env) + + view, err := gallery.DescribeVariants(models, base, env) + Expect(err).ToNot(HaveOccurred()) + Expect(view.AutoSelected).To(Equal("qwen3-8b-vllm-awq")) + }) + It("names the entry itself when no variant fits", func() { view, err := gallery.DescribeVariants(models, base, probing(gib(2), map[string]uint64{ "qwen3-8b-vllm-awq": gib(20), diff --git a/core/gallery/models.go b/core/gallery/models.go index aa5c91cce..0601f0741 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -275,11 +275,14 @@ func HostResolveEnv(ctx context.Context, systemState *system.SystemState) Resolv BackendCompatible: func(backend string) bool { return systemState.IsBackendCompatible(backend, "") }, - // The ranking half of the hardware story, from the same table that - // already picks among installed backend builds. IsBackendCompatible - // only rules out what cannot run; on a Mac both an MLX and a llama.cpp - // build can, and this is what makes the native runtime win. - BackendPreference: systemState.BackendPreferenceTokens(), + // The ranking half of the hardware story. IsBackendCompatible only rules + // out what cannot run; on a Mac both an MLX and a llama.cpp build can, + // and this is what makes the native runtime win. + // + // EnginePreferenceTokens, NOT BackendPreferenceTokens: a variant is + // matched on its gallery `backend:` engine name, and the latter reports + // build tags that no engine name contains. + EnginePreference: systemState.EnginePreferenceTokens(), ProbeMemory: func(target *GalleryModel) uint64 { return probeEntryMemory(ctx, target) }, diff --git a/core/gallery/resolve_variant.go b/core/gallery/resolve_variant.go index 6f9bf6a5c..9758dec63 100644 --- a/core/gallery/resolve_variant.go +++ b/core/gallery/resolve_variant.go @@ -66,10 +66,18 @@ type ResolveEnv struct { // A nil func treats every backend as runnable, the right default for a // caller with no view of the hardware. BackendCompatible func(backend string) bool - // BackendPreference lists the runtime tokens this host prefers, best first, - // as SystemState.BackendPreferenceTokens reports them (e.g. metal gives - // ["mlx", "metal", "cpu"], NVIDIA gives ["cuda", "vulkan", "cpu"]). A token - // is matched as a substring of a variant's backend name. + // EnginePreference lists the ENGINE NAMES this host prefers, best first, as + // SystemState.EnginePreferenceTokens reports them (e.g. NVIDIA gives + // ["vllm", "sglang", "llama-cpp"], metal gives ["mlx", "llama-cpp"]). A + // token is matched as a substring of a variant's backend name. + // + // The vocabulary is load bearing. VariantOption.Backend is a gallery entry's + // `backend:` value, which is an engine name and never carries a build tag, + // so build tags like "cuda" or "rocm" match NOTHING here. Do not wire + // SystemState.BackendPreferenceTokens into this field: that reports build + // tags for installed-build alias resolution, and the mismatch does not + // error, it scores every candidate equally and silently reduces selection to + // size alone. pkg/system/capabilities.go documents both tables. // // BackendCompatible answers "can this run here at all" and is a filter. // This answers "which of the things that CAN run here should win" and is @@ -82,7 +90,7 @@ type ResolveEnv struct { // ordering by size alone. That is the right default for a caller with no // view of the hardware, and it is what every host looked like before // preference existed. - BackendPreference []string + EnginePreference []string // ProbeMemory measures how much memory a referenced gallery entry needs, // without downloading it. A zero result means "could not tell", never // "needs nothing". @@ -103,28 +111,28 @@ func (e ResolveEnv) backendRuns(backend string) bool { return e.BackendCompatible(backend) } -// preferenceRank scores a backend against this host's preferred runtime order, +// preferenceRank scores a backend against this host's preferred engine order, // lower being better. // -// It walks BackendPreference generically and knows nothing about any particular -// backend or capability: everything a new runtime needs is expressed by the -// token table in pkg/system, so adding one never reaches this function. +// It walks EnginePreference generically and names no engine and no capability: +// everything a new runtime needs is expressed by the table in pkg/system, so +// adding one never reaches this function. // // A backend matching no token scores just below the least preferred known one. // It is never dropped, and a field where nothing matches scores uniformly, so // an unrecognised backend, an unrecognised capability and an absent preference // list all degrade to the same predictable place: ordering by size alone. func (e ResolveEnv) preferenceRank(backend string) int { - if len(e.BackendPreference) == 0 { + if len(e.EnginePreference) == 0 { return 0 } name := strings.ToLower(backend) - for i, token := range e.BackendPreference { + for i, token := range e.EnginePreference { if token != "" && strings.Contains(name, token) { return i } } - return len(e.BackendPreference) + return len(e.EnginePreference) } // VariantSelection is the outcome of a selection pass. diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index e4f72c77e..3ac9bec85 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -1,10 +1,14 @@ package gallery_test import ( + "context" + "os" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" ) var _ = Describe("VariantOption.EffectiveMemory", func() { @@ -243,18 +247,77 @@ var _ = Describe("SelectVariant", func() { }) }) - Describe("ranking by host backend preference", func() { - // The token lists below are exactly what - // SystemState.BackendPreferenceTokens reports for these hosts. They are - // spelled out rather than read from the live machine so the specs pin - // the intended behaviour on every CI runner. - darwinMetal := []string{"mlx", "metal", "cpu"} - nvidia := []string{"cuda", "vulkan", "cpu"} + Describe("ranking by host engine preference", func() { + // These are ENGINE NAMES, exactly what SystemState.EnginePreferenceTokens + // reports for these hosts and exactly what a gallery entry's `backend:` + // field holds. Build tags ("cuda", "rocm", "metal") belong to + // BackendPreferenceTokens and would match no engine name here, which is + // why every backend below is spelled as a real gallery engine. + // + // They are spelled out rather than read from the live machine so the + // specs pin the intended behaviour on every CI runner. + darwinMetal := []string{"mlx", "llama-cpp"} + nvidia := []string{"vllm", "sglang", "llama-cpp"} // A Mac runs both engines, so nothing is filtered here and preference is // the only thing that can decide. darwinRunsEverything := func(string) bool { return true } + It("prefers a vLLM build to a larger llama.cpp build on nvidia", func() { + // The rule the engine table exists to express, and the one that was + // silently inert while the ranker was fed build tags: no gallery + // engine name contains "cuda", so every candidate scored equal and + // the larger llama.cpp build won. Emptying the nvidia rule in + // pkg/system fails this spec. + options := []gallery.VariantOption{ + option("m-vllm-awq", "vllm", gib(8)), + option("m-gguf-q8", "llama-cpp", gib(24)), + base("m-gguf-q4", gib(4)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(64), + EnginePreference: nvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-vllm-awq")) + }) + + It("takes the larger llama.cpp build on the same nvidia host once preference is unknown", func() { + // The mirror of the spec above, proving the vLLM win comes from the + // preference list and not from anything intrinsic to the option set. + // This is the state the whole feature was stuck in before the two + // vocabularies were separated. + options := []gallery.VariantOption{ + option("m-vllm-awq", "vllm", gib(8)), + option("m-gguf-q8", "llama-cpp", gib(24)), + base("m-gguf-q4", gib(4)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(64), + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) + }) + + It("ranks a vllm-omni build with vllm, since the token is a substring", func() { + // Substring matching is deliberate: vllm-omni is a vLLM build and + // must inherit vLLM's rank rather than fall through to unranked. + options := []gallery.VariantOption{ + option("m-omni", "vllm-omni", gib(8)), + option("m-gguf-q8", "llama-cpp", gib(24)), + base("m-base", gib(4)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(64), + EnginePreference: nvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-omni")) + }) + It("prefers an MLX build to a larger llama.cpp build on darwin", func() { options := []gallery.VariantOption{ option("m-mlx-4bit", "mlx", gib(8)), @@ -265,13 +328,13 @@ var _ = Describe("SelectVariant", func() { selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ AvailableMemory: gib(64), BackendCompatible: darwinRunsEverything, - BackendPreference: darwinMetal, + EnginePreference: darwinMetal, }, "") Expect(err).ToNot(HaveOccurred()) Expect(selection.Option.Variant.Model).To(Equal("m-mlx-4bit")) }) - It("takes the larger llama.cpp build on the same host once preference is unknown", func() { + It("takes the larger llama.cpp build on the same darwin host once preference is unknown", func() { // The mirror of the spec above, proving the MLX win comes from the // preference list and not from anything intrinsic to the option set. options := []gallery.VariantOption{ @@ -288,75 +351,96 @@ var _ = Describe("SelectVariant", func() { Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) }) - It("prefers a CUDA build to a larger CPU-only build on nvidia", func() { - options := []gallery.VariantOption{ - option("m-cuda-q4", "cuda12-llama-cpp", gib(8)), - option("m-cpu-q8", "cpu-llama-cpp", gib(24)), - base("m-base", gib(4)), - } - - selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ - AvailableMemory: gib(64), - BackendPreference: nvidia, - }, "") - Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-cuda-q4")) - }) - - It("still picks the largest fitting build among equally preferred backends", func() { - // Preference must not flatten size ordering: with one runtime in + It("still picks the largest fitting build among equally preferred engines", func() { + // Preference must not flatten size ordering: with one engine in // play there is nothing left for it to decide. options := []gallery.VariantOption{ - option("m-cuda-q4", "cuda12-llama-cpp", gib(6)), - option("m-cuda-q8", "cuda12-llama-cpp", gib(12)), - option("m-cuda-f16", "cuda12-llama-cpp", gib(48)), + option("m-gguf-q4", "llama-cpp", gib(6)), + option("m-gguf-q8", "llama-cpp", gib(12)), + option("m-gguf-f16", "llama-cpp", gib(48)), base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ - AvailableMemory: gib(16), - BackendPreference: nvidia, + AvailableMemory: gib(16), + EnginePreference: nvidia, }, "") Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-cuda-q8")) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) }) - It("does not let a preferred backend rescue a build that does not fit", func() { + It("does not let a preferred engine rescue a build that does not fit", func() { // Fit is a filter and preference is only a ranking among survivors. - // The CUDA build is both preferred and too large, so the CPU build - // the host can actually hold has to win. + // The vLLM build is both preferred and too large, so the llama.cpp + // build the host can actually hold has to win. options := []gallery.VariantOption{ - option("m-cuda-f16", "cuda12-llama-cpp", gib(48)), - option("m-cpu-q4", "cpu-llama-cpp", gib(6)), + option("m-vllm-fp16", "vllm", gib(48)), + option("m-gguf-q4", "llama-cpp", gib(6)), base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ - AvailableMemory: gib(16), - BackendPreference: nvidia, + AvailableMemory: gib(16), + EnginePreference: nvidia, }, "") Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-cpu-q4")) - Expect(selection.Reasons).To(ContainElement(ContainSubstring("m-cuda-f16"))) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q4")) + Expect(selection.Reasons).To(ContainElement(ContainSubstring("m-vllm-fp16"))) }) - It("orders unrecognised backends by size rather than dropping them", func() { - // Neither engine name carries an nvidia token. Nothing is discarded + It("orders engines absent from the table by size rather than dropping them", func() { + // Neither engine appears in the nvidia rule. Nothing is discarded // and nothing is arbitrarily favoured; the host falls back to the // size-only behaviour it had before preference existed. + // + // The base is left unsized so it ranks in the tier below a proven + // fit. It is a llama.cpp build, which the nvidia rule DOES rank, and + // letting it compete in the same tier would prove preference works + // rather than proving unlisted engines degrade to size. options := []gallery.VariantOption{ - option("m-vllm-awq", "vllm", gib(12)), - option("m-sglang-fp8", "sglang", gib(6)), + option("m-diffusers", "diffusers", gib(12)), + option("m-transformers", "transformers", gib(6)), + base("m-base", 0), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(16), + EnginePreference: nvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-diffusers")) + Expect(selection.Reasons).To(BeEmpty()) + }) + + It("ranks sglang between vllm and llama-cpp on nvidia", func() { + // A judgement call worth pinning: sglang is a GPU serving engine of + // the same class as vllm, so it outranks the portable engine, but it + // sits behind vllm. + options := []gallery.VariantOption{ + option("m-vllm", "vllm", gib(4)), + option("m-sglang", "sglang", gib(6)), + option("m-gguf", "llama-cpp", gib(8)), base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ - AvailableMemory: gib(16), - BackendPreference: nvidia, + AvailableMemory: gib(16), + EnginePreference: nvidia, }, "") Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-vllm-awq")) - Expect(selection.Reasons).To(BeEmpty()) + Expect(selection.Option.Variant.Model).To(Equal("m-vllm")) + + withoutVLLM := []gallery.VariantOption{ + option("m-sglang", "sglang", gib(6)), + option("m-gguf", "llama-cpp", gib(8)), + base("m-base", gib(2)), + } + selection, err = gallery.SelectVariant(withoutVLLM, gallery.ResolveEnv{ + AvailableMemory: gib(16), + EnginePreference: nvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-sglang")) }) It("still installs the base when nothing fits, whatever the host prefers", func() { @@ -369,7 +453,7 @@ var _ = Describe("SelectVariant", func() { selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ AvailableMemory: gib(4), BackendCompatible: darwinRunsEverything, - BackendPreference: darwinMetal, + EnginePreference: darwinMetal, }, "") Expect(err).ToNot(HaveOccurred()) Expect(selection.Option.Variant.Model).To(Equal("m-base")) @@ -389,7 +473,7 @@ var _ = Describe("SelectVariant", func() { selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ AvailableMemory: gib(64), BackendCompatible: darwinRunsEverything, - BackendPreference: darwinMetal, + EnginePreference: darwinMetal, }, "m-gguf-q8") Expect(err).ToNot(HaveOccurred()) Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) @@ -502,3 +586,79 @@ var _ = Describe("SelectVariant", func() { }) }) }) + +var _ = Describe("HostResolveEnv engine preference wiring", func() { + // The specs above feed SelectVariant a hand-written token list, which proves + // the ranker but not that the host actually reaches the engine table. These + // drive the REAL table through the REAL wiring, so emptying + // engineNamePreferenceRules in pkg/system, or reverting this field to + // BackendPreferenceTokens, fails here. + var origEnv string + const capabilityEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY" + + BeforeEach(func() { + origEnv = os.Getenv(capabilityEnv) + }) + + AfterEach(func() { + if origEnv != "" { + os.Setenv(capabilityEnv, origEnv) + } else { + os.Unsetenv(capabilityEnv) + } + }) + + envFor := func(capability string) gallery.ResolveEnv { + GinkgoHelper() + Expect(os.Setenv(capabilityEnv, capability)).To(Succeed()) + return gallery.HostResolveEnv(context.Background(), &system.SystemState{}) + } + + It("hands the ranker engine names an NVIDIA host's gallery entries can match", func() { + preference := envFor("nvidia-cuda-12").EnginePreference + Expect(preference).To(Equal([]string{"vllm", "sglang", "llama-cpp"})) + }) + + It("installs the vLLM build over a larger llama.cpp one on a real NVIDIA host", func() { + // End to end on the live table: the exact behaviour that was silently + // dead while the ranker was fed build tags. + env := envFor("nvidia-cuda-12") + env.AvailableMemory = 64 * 1024 * 1024 * 1024 + + options := []gallery.VariantOption{ + {Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 8 * 1024 * 1024 * 1024}, + {Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 24 * 1024 * 1024 * 1024}, + } + + selection, err := gallery.SelectVariant(options, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-vllm")) + }) + + It("installs the MLX build over a larger llama.cpp one on a real darwin host", func() { + env := envFor("metal") + env.AvailableMemory = 64 * 1024 * 1024 * 1024 + // The real gate rejects mlx off darwin, and this spec is about ranking. + env.BackendCompatible = func(string) bool { return true } + + options := []gallery.VariantOption{ + {Variant: gallery.Variant{Model: "m-mlx"}, Backend: "mlx", ProbedMemory: 8 * 1024 * 1024 * 1024}, + {Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 24 * 1024 * 1024 * 1024}, + } + + selection, err := gallery.SelectVariant(options, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-mlx")) + }) + + It("carries no build tag into the ranker, whatever the host", func() { + // The wiring-level lock. BackendPreferenceTokens returns build tags for + // every capability, so if it is ever wired back into this field, one of + // these tags shows up here. + for _, capability := range []string{"nvidia-cuda-12", "amd", "intel", "metal", "vulkan", "default"} { + Expect(envFor(capability).EnginePreference).ToNot( + ContainElements("cuda", "rocm", "hip", "sycl", "metal", "cpu", "darwin-x86"), + "capability %q leaked a build tag into the variant ranker", capability) + } + }) +}) diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 6aaf85105..172283248 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -45,19 +45,47 @@ const ( backendTokenHIP = "hip" backendTokenSYCL = "sycl" backendTokenCPU = "cpu" + + // Engine names (private). Unlike the tokens above these are whole backend + // identities as a gallery entry's `backend:` field spells them, not build + // tags. See the two preference tables below for why the distinction matters. + engineVLLM = "vllm" + engineSGLang = "sglang" + engineLlamaCpp = "llama-cpp" + engineMLX = "mlx" ) -// backendPreferenceRule maps a detected capability to the runtime tokens that -// capability prefers, best first. +// There are TWO preference tables below and they speak DIFFERENT VOCABULARIES. +// Merging them looks tempting and silently breaks one of the two consumers, +// because a token that means something in one vocabulary means nothing in the +// other. Read this before editing either. // -// This table is the single source of truth for "what should this host run -// first", consumed both by concrete-backend alias resolution and by gallery -// variant auto-selection. Keeping it declarative is deliberate: LocalAI gains -// backends and hardware targets continuously, and expressing a new preference -// has to stay a one-line edit here rather than a change to any ranking code. +// - backendBuildTagPreferenceRules holds BUILD TAGS ("cuda", "rocm", "metal"). +// They are matched against INSTALLED BACKEND BUILD DIRECTORY NAMES such as +// "llama-cpp-cuda-12" or "cuda12-vllm". Consumer: alias resolution in +// ListSystemBackends (core/gallery/backends.go), which picks which installed +// build of one alias to run. // -// To add a capability, append a rule. To reorder one host's runtimes, reorder -// its tokens. Nothing else needs to change. +// - engineNamePreferenceRules holds ENGINE NAMES ("vllm", "llama-cpp", "mlx"). +// They are matched against a gallery entry's `backend:` value, which never +// carries a build tag: no entry in gallery/index.yaml contains "cuda", +// "rocm", "sycl" or "vulkan" anywhere in its backend name. Consumer: gallery +// variant auto-selection (core/gallery/resolve_variant.go), which picks +// which build of one model's weights to install. +// +// Feeding build tags to the variant ranker matches nothing, which does not +// error: every candidate simply scores equal and size alone decides, so the +// preference silently stops existing. That is exactly the bug this split fixes. + +// backendPreferenceRule maps a detected capability to preferred tokens, best +// first. Both tables share this shape; only their vocabulary differs. +type backendPreferenceRule struct { + capabilityPrefix string + tokens []string +} + +// backendBuildTagPreferenceRules is the BUILD TAG table. See the block comment +// above for the vocabulary contract. // // Matching is by capability PREFIX, because a detected capability is refined at // runtime ("nvidia" becomes "nvidia-cuda-12" when the toolkit is present) and a @@ -65,31 +93,62 @@ const ( // in order, so a more specific prefix must precede any rule it shares a prefix // with. // -// Tokens are matched as substrings of a backend name, so "cuda" covers -// "cuda12-llama-cpp" and "mlx" covers "metal-mlx" alike. A backend matching no -// token is not an error and is never discarded; it simply sorts below every -// recognised one. -type backendPreferenceRule struct { - capabilityPrefix string - tokens []string -} - -var backendPreferenceRules = []backendPreferenceRule{ +// Tokens are matched as substrings of a build directory name, so "cuda" covers +// "cuda12-llama-cpp". A build matching no token is not an error and is never +// discarded; it simply sorts below every recognised one. +var backendBuildTagPreferenceRules = []backendPreferenceRule{ {Nvidia, []string{backendTokenCUDA, vulkan, backendTokenCPU}}, {AMD, []string{backendTokenROCM, backendTokenHIP, vulkan, backendTokenCPU}}, {Intel, []string{backendTokenSYCL, Intel, backendTokenCPU}}, - // MLX outranks metal because on Apple silicon it is the native accelerated - // runtime, whereas a metal-enabled GGUF build is the portable engine merely - // compiled with GPU offload. - {metal, []string{backendTokenMLX, backendTokenMetal, backendTokenCPU}}, + {metal, []string{backendTokenMetal, backendTokenCPU}}, {darwinX86, []string{darwinX86, backendTokenCPU}}, {vulkan, []string{vulkan, backendTokenCPU}}, } -// defaultBackendPreferenceTokens is what a host with no matching rule prefers. +// defaultBackendBuildTagTokens is what a host with no matching rule prefers. // A capability nobody has taught this table about degrades to plain CPU rather // than to an error or to an empty list. -var defaultBackendPreferenceTokens = []string{backendTokenCPU} +var defaultBackendBuildTagTokens = []string{backendTokenCPU} + +// engineNamePreferenceRules is the ENGINE NAME table. See the block comment +// above for the vocabulary contract. +// +// Capability matching is by prefix, exactly as the build tag table does it. +// +// Tokens are matched as SUBSTRINGS of an engine name, which is load bearing: +// "vllm" also covers "vllm-omni", "mlx" also covers "mlx-vlm" and "mlx-audio", +// and "llama-cpp" also covers "ik-llama-cpp". Each of those is a build of the +// engine named by the token, so ranking them together is correct. Order the +// tokens so no token is a substring of an engine that should rank differently. +// +// A capability is deliberately ABSENT rather than guessed at when no engine +// ordering can be justified for it. An absent rule degrades to ordering by size +// alone, which is the behaviour that predates preference and is always safe. +// darwin-x86 is absent for that reason: nothing accelerates there, so no engine +// deserves to outrank another. +var engineNamePreferenceRules = []backendPreferenceRule{ + // vLLM first on every host with a dedicated serving engine build. It is the + // throughput engine, and a model published with a vLLM build is published + // that way precisely because that build is the one worth running. + // SGLang sits directly behind it: same class of GPU serving engine, ships + // cuda/rocm/intel builds alike, but it is behind vLLM because vLLM covers + // far more of the gallery. llama-cpp is the portable fallback. + {Nvidia, []string{engineVLLM, engineSGLang, engineLlamaCpp}}, + {AMD, []string{engineVLLM, engineSGLang, engineLlamaCpp}}, + {Intel, []string{engineVLLM, engineSGLang, engineLlamaCpp}}, + // MLX is the native accelerated runtime on Apple silicon, whereas a + // metal-enabled GGUF build is the portable engine merely compiled with GPU + // offload. No vLLM or SGLang build targets metal, so neither is listed. + {metal, []string{engineMLX, engineLlamaCpp}}, + // A Vulkan host has exactly one LLM engine with a Vulkan build, so a + // llama-cpp variant is the only one that will use the GPU at all. + {vulkan, []string{engineLlamaCpp}}, +} + +// defaultEnginePreferenceTokens is empty on purpose. A host with no detected +// accelerator has no reason to prefer one engine over another, and an empty +// list is what the ranker reads as "order by size alone". +var defaultEnginePreferenceTokens = []string{} var ( cuda13DirExists bool @@ -232,18 +291,40 @@ func (s *SystemState) getSystemCapabilities() string { // backend implementation order for the current system capability. Callers can use // these tokens to select the most appropriate concrete backend among multiple // candidates sharing the same alias (e.g., "llama-cpp"). -// The rules live in backendPreferenceRules above; this function only looks them -// up, so teaching LocalAI about a new runtime never means editing logic. +// +// These are BUILD TAGS matched against installed build directory names. For +// engine names as a gallery entry spells them, use EnginePreferenceTokens. func (s *SystemState) BackendPreferenceTokens() []string { + return s.preferenceTokens(backendBuildTagPreferenceRules, defaultBackendBuildTagTokens) +} + +// EnginePreferenceTokens returns the engine names this host prefers, best first, +// for ranking gallery model variants against each other. +// +// These are ENGINE NAMES matched against a gallery entry's `backend:` value +// ("vllm", "llama-cpp", "mlx"), never build tags. Feeding this function's output +// to installed-build alias resolution, or BackendPreferenceTokens' output to +// variant ranking, matches nothing and silently disables the preference. +// +// An empty result is normal and means "no engine ordering applies here", which +// the ranker reads as ordering by size alone. +func (s *SystemState) EnginePreferenceTokens() []string { + return s.preferenceTokens(engineNamePreferenceRules, defaultEnginePreferenceTokens) +} + +// preferenceTokens resolves the current capability against one preference table. +// The rules live in the tables above; this only looks them up, so teaching +// LocalAI about a new runtime never means editing logic. +func (s *SystemState) preferenceTokens(rules []backendPreferenceRule, fallback []string) []string { capStr := strings.ToLower(s.getSystemCapabilities()) - for _, rule := range backendPreferenceRules { + for _, rule := range rules { if strings.HasPrefix(capStr, rule.capabilityPrefix) { // Copied so a caller cannot mutate the shared table out from under // every other host lookup. return slices.Clone(rule.tokens) } } - return slices.Clone(defaultBackendPreferenceTokens) + return slices.Clone(fallback) } // DetectedCapability returns the raw detected capability string (e.g. "metal", diff --git a/pkg/system/capabilities_test.go b/pkg/system/capabilities_test.go index eb3c2e70c..9ac30b991 100644 --- a/pkg/system/capabilities_test.go +++ b/pkg/system/capabilities_test.go @@ -149,22 +149,36 @@ var _ = Describe("BackendPreferenceTokens", func() { return (&SystemState{}).BackendPreferenceTokens() } - It("ranks MLX above metal on apple silicon", func() { - // MLX is the native accelerated runtime there, so it must outrank a - // metal-enabled build of the portable engine. - Expect(tokensFor(metal)).To(Equal([]string{"mlx", "metal", "cpu"})) - }) + // This table is a REGRESSION LOCK, not a description. These are build tags + // matched against installed backend build directory names, and the only + // consumer is alias resolution in ListSystemBackends. Every value below is + // the output this function had before engine preference existed. + // + // Engine names ("vllm", "mlx", "llama-cpp") belong to EnginePreferenceTokens + // and MUST NOT appear here. Merging the two vocabularies is exactly the + // mistake this table exists to catch: it does not error, it silently makes + // one of the two consumers rank everything equally. + DescribeTable("returns the original build tags for every capability", + func(capability string, want []string) { + Expect(tokensFor(capability)).To(Equal(want)) + }, + Entry("nvidia", "nvidia-cuda-12", []string{"cuda", "vulkan", "cpu"}), + Entry("amd", AMD, []string{"rocm", "hip", "vulkan", "cpu"}), + Entry("intel", Intel, []string{"sycl", "intel", "cpu"}), + Entry("metal", metal, []string{"metal", "cpu"}), + Entry("darwin-x86", darwinX86, []string{"darwin-x86", "cpu"}), + Entry("vulkan", vulkan, []string{"vulkan", "cpu"}), + Entry("unknown capability", "some-future-accelerator", []string{"cpu"}), + ) - It("keeps the vendor order every other capability already had", func() { - Expect(tokensFor("nvidia-cuda-12")).To(Equal([]string{"cuda", "vulkan", "cpu"})) - Expect(tokensFor(AMD)).To(Equal([]string{"rocm", "hip", "vulkan", "cpu"})) - Expect(tokensFor(Intel)).To(Equal([]string{"sycl", "intel", "cpu"})) - Expect(tokensFor(darwinX86)).To(Equal([]string{"darwin-x86", "cpu"})) - Expect(tokensFor(vulkan)).To(Equal([]string{"vulkan", "cpu"})) - }) - - It("degrades an unknown capability to cpu rather than to nothing", func() { - Expect(tokensFor("some-future-accelerator")).To(Equal([]string{"cpu"})) + It("carries no engine name in any capability's tokens", func() { + // The generic form of the lock above: whatever anyone adds to the build + // tag table later, an engine name in it is a merged vocabulary. + engineNames := []string{"vllm", "sglang", "llama-cpp", "mlx"} + for _, capability := range []string{"nvidia", AMD, Intel, metal, darwinX86, vulkan, "default"} { + Expect(tokensFor(capability)).ToNot(ContainElements(engineNames), + "capability %q leaked an engine name into the build tag table", capability) + } }) It("matches a capability by prefix so a refined one keeps its vendor order", func() { @@ -178,6 +192,75 @@ var _ = Describe("BackendPreferenceTokens", func() { }) }) +var _ = Describe("EnginePreferenceTokens", func() { + var origEnv string + + BeforeEach(func() { + origEnv = os.Getenv(capabilityEnv) + }) + + AfterEach(func() { + if origEnv != "" { + os.Setenv(capabilityEnv, origEnv) + } else { + os.Unsetenv(capabilityEnv) + } + }) + + tokensFor := func(capability string) []string { + GinkgoHelper() + Expect(os.Setenv(capabilityEnv, capability)).To(Succeed()) + return (&SystemState{}).EnginePreferenceTokens() + } + + It("puts vLLM ahead of llama.cpp on nvidia", func() { + // The rule the whole engine table exists to express. A build tag list + // here would match no engine name at all and this would read empty. + Expect(tokensFor("nvidia-cuda-12")).To(Equal([]string{"vllm", "sglang", "llama-cpp"})) + }) + + It("puts MLX ahead of llama.cpp on apple silicon", func() { + Expect(tokensFor(metal)).To(Equal([]string{"mlx", "llama-cpp"})) + }) + + It("gives the GPU serving engines to every vendor that has builds of them", func() { + Expect(tokensFor(AMD)).To(Equal([]string{"vllm", "sglang", "llama-cpp"})) + Expect(tokensFor(Intel)).To(Equal([]string{"vllm", "sglang", "llama-cpp"})) + }) + + It("prefers llama.cpp on vulkan, the only engine with a vulkan build", func() { + Expect(tokensFor(vulkan)).To(Equal([]string{"llama-cpp"})) + }) + + It("names only engines, never a build tag", func() { + // The mirror of the lock on BackendPreferenceTokens. A build tag here + // matches no gallery `backend:` value and silently disables ranking. + buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86"} + for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan} { + Expect(tokensFor(capability)).ToNot(ContainElements(buildTags), + "capability %q leaked a build tag into the engine table", capability) + } + }) + + It("leaves a capability with no justified engine order empty rather than guessing", func() { + // Empty is read by the variant ranker as "order by size alone", which is + // the behaviour that predates preference and is always safe. + Expect(tokensFor(darwinX86)).To(BeEmpty()) + Expect(tokensFor("default")).To(BeEmpty()) + Expect(tokensFor("some-future-accelerator")).To(BeEmpty()) + }) + + It("matches a capability by prefix so a refined one keeps its engine order", func() { + Expect(tokensFor("nvidia-l4t-cuda-13")).To(Equal(tokensFor("nvidia"))) + }) + + It("hands out a copy so one caller cannot corrupt another's lookup", func() { + first := tokensFor("nvidia") + first[0] = "clobbered" + Expect(tokensFor("nvidia")[0]).To(Equal("vllm")) + }) +}) + var _ = Describe("CapabilityFilterDisabled", func() { var origEnv string