fix(gallery): prefer llama.cpp over GPU serving engines on hosts with no GPU

engineNamePreferenceRules had no row for the "default" capability, which
getSystemCapabilities() returns both when no GPU is detected and when a GPU
is present but under the 4 GiB VRAM floor. A missing row yields an empty
preference list, which preferenceRank reads as "score everything equally",
collapsing variant selection to size alone.

That would be harmless if the hardware filter dropped GPU serving engines on
such a host, but it does not. IsBackendCompatible derives support from the
engine NAME, and "vllm" and "sglang" contain none of the darwin, cuda, rocm
or sycl tokens it keys on, so they fall through to its closing "return true".
A vLLM variant therefore survives on a CPU-only box and wins whenever its
build is the larger of the two on offer: the machine installs vLLM in
preference to llama.cpp.

darwin-x86 had the identical hole. It was documented as a deliberate omission
because nothing accelerates on an Intel Mac, which is true about acceleration
and wrong about consequence: with every engine tied, download size decides.

Add rows for both putting llama-cpp first. The GPU engines are enumerated
behind it rather than left unmatched: an unmatched engine already ranks below
every listed one, so llama.cpp would win either way, but unmatched engines
also tie with each other and let size decide among them. Naming them fixes
that order. MLX is left off the darwin-x86 row on purpose so it ranks last,
since IsBackendCompatible admits darwin-tokened engines on that capability
even though MLX needs Apple silicon.

Preference orders survivors and never filters, so a model published only as a
vLLM build is still installed on a host with no GPU; there is a spec for it.

Surveyed every other value getSystemCapabilities() can return. nvidia, amd,
intel and vulkan have rows; the l4t and cuda-refined values reach the nvidia
row by prefix; "apple" and "" cannot reach the vendor fallthrough because the
darwin and no-GPU branches return earlier. default and darwin-x86 were the
only live holes.

BackendPreferenceTokens and its build-tag table are untouched, and
preferenceRank stays generic, naming no engine and no capability.

Assisted-by: Claude:claude-opus-4-8
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.agents/adding-backends.md

@@ -244,6 +244,18 @@ Leaving your backend out is a valid choice when no ordering can be justified for
 it. It then ranks below every known engine and selection falls back to size,
 which is the behaviour that predates preference.
 
+**Leaving a whole capability out is not.** A missing row gives that host an
+empty preference list, so size alone decides among everything that survives the
+filters, and the filter will not save you: `IsBackendCompatible` derives hardware
+support from the engine NAME, so `vllm` and `sglang` carry no darwin, cuda, rocm
+or sycl token and are never dropped on a host with no GPU. That is why `default`
+(no usable accelerator, including a GPU under the 4 GiB VRAM floor) and
+`darwin-x86` both have rows putting `llama-cpp` first. Every capability
+`getSystemCapabilities()` can return needs a row unless every engine really is
+equally at home there. When you add one, enumerate the engines you are demoting
+rather than relying on them falling through unmatched: unmatched engines all tie
+with each other, so size decides among them.
+
 ## Documenting the backend (README + docs)
 
 A backend is not "added" until it is discoverable. Update the user-facing docs:

core/gallery/resolve_variant_test.go

@@ -3,6 +3,8 @@ package gallery_test
 import (
 	"context"
 	"os"
+	"path/filepath"
+	"runtime"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -593,11 +595,15 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() {
 	// drive the REAL table through the REAL wiring, so emptying
 	// engineNamePreferenceRules in pkg/system, or reverting this field to
 	// BackendPreferenceTokens, fails here.
-	var origEnv string
+	var origEnv, origRunFileEnv string
 	const capabilityEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY"
+	const capabilityRunFileEnv = "LOCALAI_FORCE_META_BACKEND_CAPABILITY_RUN_FILE"
+	// What getSystemCapabilities reports for a host with no usable accelerator.
+	const noGPUCapability = "default"
 
 	BeforeEach(func() {
 		origEnv = os.Getenv(capabilityEnv)
+		origRunFileEnv = os.Getenv(capabilityRunFileEnv)
 	})
 
 	AfterEach(func() {
@@ -606,6 +612,11 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() {
 		} else {
 			os.Unsetenv(capabilityEnv)
 		}
+		if origRunFileEnv != "" {
+			os.Setenv(capabilityRunFileEnv, origRunFileEnv)
+		} else {
+			os.Unsetenv(capabilityRunFileEnv)
+		}
 	})
 
 	envFor := func(capability string) gallery.ResolveEnv {
@@ -651,6 +662,80 @@ var _ = Describe("HostResolveEnv engine preference wiring", func() {
 		Expect(selection.Option.Variant.Model).To(Equal("m-mlx"))
 	})
 
+	It("installs the llama.cpp build over a larger vLLM one on a host with no GPU", func() {
+		// The hole this rule closes. IsBackendCompatible keys on the engine
+		// name, and "vllm" carries no darwin/cuda/rocm/sycl token, so a vLLM
+		// build is NOT filtered out here. Emptying the default rule in
+		// pkg/system puts the larger vLLM build back on a CPU-only box.
+		env := envFor(noGPUCapability)
+		env.AvailableMemory = 64 * 1024 * 1024 * 1024
+
+		options := []gallery.VariantOption{
+			{Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 24 * 1024 * 1024 * 1024},
+			{Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 8 * 1024 * 1024 * 1024},
+		}
+
+		selection, err := gallery.SelectVariant(options, env, "")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(selection.Option.Variant.Model).To(Equal("m-gguf"))
+	})
+
+	It("installs the llama.cpp build over a larger vLLM one on an intel mac", func() {
+		env := envFor("darwin-x86")
+		env.AvailableMemory = 64 * 1024 * 1024 * 1024
+
+		options := []gallery.VariantOption{
+			{Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 24 * 1024 * 1024 * 1024},
+			{Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 8 * 1024 * 1024 * 1024},
+		}
+
+		selection, err := gallery.SelectVariant(options, env, "")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(selection.Option.Variant.Model).To(Equal("m-gguf"))
+	})
+
+	It("still installs a vLLM build on a host with no GPU when it is the only one offered", func() {
+		// Preference ORDERS survivors, it never filters them. Demoting vLLM must
+		// not make a model published only as a vLLM build uninstallable.
+		env := envFor(noGPUCapability)
+		env.AvailableMemory = 64 * 1024 * 1024 * 1024
+
+		options := []gallery.VariantOption{
+			{Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 8 * 1024 * 1024 * 1024},
+		}
+
+		selection, err := gallery.SelectVariant(options, env, "")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(selection.Option.Variant.Model).To(Equal("m-vllm"))
+	})
+
+	It("prefers llama.cpp on a GPU host with too little VRAM to serve from", func() {
+		// This host has a GPU, yet getSystemCapabilities reports "default"
+		// because it is under the 4 GiB floor. Driven through the real detector
+		// rather than a forced capability, so that mapping is exercised too.
+		if runtime.GOOS == "darwin" {
+			Skip("darwin reports metal or darwin-x86 before the VRAM floor is consulted")
+		}
+		Expect(os.Unsetenv(capabilityEnv)).To(Succeed())
+		// A capability run file on the machine would override detection.
+		Expect(os.Setenv(capabilityRunFileEnv, filepath.Join(GinkgoT().TempDir(), "absent"))).To(Succeed())
+
+		state := &system.SystemState{GPUVendor: system.Nvidia, VRAM: 2 * 1024 * 1024 * 1024}
+		Expect(state.DetectedCapability()).To(Equal(noGPUCapability))
+
+		env := gallery.HostResolveEnv(context.Background(), state)
+		env.AvailableMemory = 64 * 1024 * 1024 * 1024
+
+		options := []gallery.VariantOption{
+			{Variant: gallery.Variant{Model: "m-vllm"}, Backend: "vllm", ProbedMemory: 24 * 1024 * 1024 * 1024},
+			{Variant: gallery.Variant{Model: "m-gguf"}, Backend: "llama-cpp", ProbedMemory: 8 * 1024 * 1024 * 1024},
+		}
+
+		selection, err := gallery.SelectVariant(options, env, "")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(selection.Option.Variant.Model).To(Equal("m-gguf"))
+	})
+
 	It("carries no build tag into the ranker, whatever the host", func() {
 		// The wiring-level lock. BackendPreferenceTokens returns build tags for
 		// every capability, so if it is ever wired back into this field, one of

pkg/system/capabilities.go

@@ -124,8 +124,14 @@ var defaultBackendBuildTagTokens = []string{backendTokenCPU}
 // A capability is deliberately ABSENT rather than guessed at when no engine
 // ordering can be justified for it. An absent rule degrades to ordering by size
 // alone, which is the behaviour that predates preference and is always safe.
-// darwin-x86 is absent for that reason: nothing accelerates there, so no engine
-// deserves to outrank another.
+//
+// Safe, however, only where every candidate engine is equally at home on the
+// host. Where one is not, the rule has to be written here, because the hardware
+// filter will not write it: IsBackendCompatible derives support from the engine
+// NAME, and "vllm" and "sglang" contain none of the darwin, cuda, rocm or sycl
+// tokens it keys on, so a GPU serving engine is never filtered out on a host
+// with no GPU. Left unranked there, it wins on size alone whenever its build is
+// the larger one, and a CPU-only box installs vLLM over llama.cpp.
 var engineNamePreferenceRules = []backendPreferenceRule{
 	// vLLM first on every host with a dedicated serving engine build. It is the
 	// throughput engine, and a model published with a vLLM build is published
@@ -143,11 +149,35 @@ var engineNamePreferenceRules = []backendPreferenceRule{
 	// A Vulkan host has exactly one LLM engine with a Vulkan build, so a
 	// llama-cpp variant is the only one that will use the GPU at all.
 	{vulkan, []string{engineLlamaCpp}},
+	// No usable accelerator: either nothing was detected, or a GPU was found
+	// with too little VRAM to serve from, both of which report "default".
+	// llama.cpp is the engine built to run well on a CPU; vLLM and SGLang are
+	// GPU serving engines whose CPU paths exist but are not what an unattended
+	// auto-selection should hand a CPU-only box.
+	//
+	// The GPU engines are enumerated behind llama.cpp rather than left
+	// unmatched. Listing llama.cpp alone would already make it win, since an
+	// unmatched engine ranks below every listed one, but it would leave vLLM
+	// and SGLang tied with each other and with any engine nobody has ranked
+	// yet, so download size would decide among them. Naming them fixes that
+	// order and says the omission of a GPU engine from the top spot is a
+	// decision rather than an oversight. Ranking never filters, so a vLLM
+	// variant offered alone is still installed here.
+	{defaultCapability, []string{engineLlamaCpp, engineVLLM, engineSGLang}},
+	// An Intel Mac has no accelerated runtime at all: MLX needs Apple silicon,
+	// and no vLLM or SGLang build targets darwin. Same ordering, same reasons.
+	// MLX is deliberately left unlisted so it ranks last, since
+	// IsBackendCompatible admits any darwin-tokened engine on this capability
+	// and will not drop it.
+	{darwinX86, []string{engineLlamaCpp, engineVLLM, engineSGLang}},
 }
 
-// defaultEnginePreferenceTokens is empty on purpose. A host with no detected
-// accelerator has no reason to prefer one engine over another, and an empty
-// list is what the ranker reads as "order by size alone".
+// defaultEnginePreferenceTokens is empty on purpose. It is now only reached by
+// a capability nobody has taught this table about, which an operator can force
+// via LOCALAI_FORCE_META_BACKEND_CAPABILITY; the detected "default" has its own
+// rule above. Guessing an order for hardware we know nothing about would be
+// worse than ordering by size alone, which is what the ranker reads an empty
+// list as and is the behaviour that predates preference.
 var defaultEnginePreferenceTokens = []string{}
 
 var (

pkg/system/capabilities_test.go

@@ -236,17 +236,37 @@ var _ = Describe("EnginePreferenceTokens", func() {
 		// The mirror of the lock on BackendPreferenceTokens. A build tag here
 		// matches no gallery `backend:` value and silently disables ranking.
 		buildTags := []string{"cuda", "rocm", "hip", "sycl", "vulkan", "metal", "cpu", "darwin-x86"}
-		for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan} {
+		for _, capability := range []string{"nvidia", AMD, Intel, metal, vulkan, defaultCapability, darwinX86} {
 			Expect(tokensFor(capability)).ToNot(ContainElements(buildTags),
 				"capability %q leaked a build tag into the engine table", capability)
 		}
 	})
 
-	It("leaves a capability with no justified engine order empty rather than guessing", func() {
+	It("prefers llama.cpp on a host with no usable accelerator", func() {
+		// Nothing filters a GPU serving engine out here: IsBackendCompatible
+		// keys on the engine name, and "vllm" carries no hardware token. Without
+		// this rule a larger vLLM build would win a CPU-only host on size.
+		Expect(tokensFor(defaultCapability)).To(Equal([]string{"llama-cpp", "vllm", "sglang"}))
+	})
+
+	It("prefers llama.cpp on an intel mac, where nothing accelerates", func() {
+		Expect(tokensFor(darwinX86)).To(Equal([]string{"llama-cpp", "vllm", "sglang"}))
+	})
+
+	It("ranks the GPU serving engines below llama.cpp rather than leaving them tied", func() {
+		// Enumerating them is what stops download size deciding between vLLM and
+		// SGLang once llama.cpp is out of the running.
+		for _, capability := range []string{defaultCapability, darwinX86} {
+			tokens := tokensFor(capability)
+			Expect(tokens).To(HaveLen(3), "capability %q", capability)
+			Expect(tokens[0]).To(Equal("llama-cpp"), "capability %q", capability)
+		}
+	})
+
+	It("leaves a capability nobody has taught the table about empty rather than guessing", func() {
 		// Empty is read by the variant ranker as "order by size alone", which is
-		// the behaviour that predates preference and is always safe.
-		Expect(tokensFor(darwinX86)).To(BeEmpty())
-		Expect(tokensFor("default")).To(BeEmpty())
+		// the behaviour that predates preference and is always safe. Only a
+		// forced, unrecognised capability lands here now.
 		Expect(tokensFor("some-future-accelerator")).To(BeEmpty())
 	})