fix(ci): treat .docker/ as a shared build input

#11346 changed how ROCm llama.cpp compiles and built nothing. It touched
.docker/llama-cpp-build-target.sh and scripts/build/llama-cpp-build-target_test.sh,
no SHARED_BUILD_INPUTS rule matched either (the second is carved out on
purpose), so the filter selected zero entries and all six backend job
groups reported "skipping". The fix shipped to master unvalidated.

.docker/ holds the per-backend compile and build-target scripts for
llama-cpp, turboquant, bonsai and ik-llama-cpp, plus inputs every
Dockerfile consumes (apt-mirror.sh, install-base-deps.sh). Those files
decide how a backend is compiled, so a change to one can only be
validated by rebuilding it.

Take the same posture the scripts/build/ catch-all already takes, and for
the reason its comment already gives: a shared build input silently
shipping to nothing is the failure this list exists to prevent. A rule
sees only the matrix entry, not which file matched it, so narrowing
.docker/<name>-compile.sh to the backend named by its prefix would mean
threading the filename through matchedSharedRules. Not worth it for files
edited a handful of times a release.

Two regression tests, both verified red against the unfixed filter:
one asserting a .docker/ compile script selects the backend it compiles,
one asserting a shared .docker/ input takes the full matrix.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude Code:claude-opus-5 [Read] [Edit] [Bash]

.docker/llama-cpp-build-target.sh

@@ -8,8 +8,15 @@ build_type=${2-}
 # ggml-cpu/arch/x86/repack.cpp at -march=sapphirerapids: the job sits on that one
 # translation unit until GitHub kills it at 6h. gcc builds the same file in
 # seconds, so only the SYCL images have to give up the CPU variant matrix.
+#
+# ROCm runs out of the same 6h budget for a different reason: volume, not a
+# stall. hipcc compiles ggml's HIP kernels once per entry in AMDGPU_TARGETS,
+# which is eleven architectures (gfx908 through gfx1201), and the CPU variant
+# matrix lands on top of that. The job built in 2h27m before it was added and
+# has been killed at exactly 6h00m on every run since, so no ROCm llama-cpp
+# image has been published since 2026-08-01.
 case "$build_type" in
-  sycl*)
+  sycl*|hipblas*)
     echo llama-cpp-fallback
     exit 0
     ;;

backend/cpp/ds4/CMakeLists.txt

@@ -69,7 +69,15 @@ target_include_directories(hw_grpc_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
 
 set(DS4_OBJS "${DS4_DIR}/ds4.o")
 if(DS4_GPU STREQUAL "cuda")
-    list(APPEND DS4_OBJS "${DS4_DIR}/ds4_cuda.o")
+    list(APPEND DS4_OBJS
+        "${DS4_DIR}/ds4_cuda.o"
+        "${DS4_DIR}/cuda/mmq/ds4_ggml_stubs.o"
+        "${DS4_DIR}/cuda/mmq/ds4_mmq.o"
+        "${DS4_DIR}/cuda/mmq/ds4_mmq_d2r.o"
+        "${DS4_DIR}/cuda/mmq/quantize.o"
+        "${DS4_DIR}/cuda/mmq/mmid.o"
+        "${DS4_DIR}/cuda/mmq/mmvq.o"
+        "${DS4_DIR}/cuda/mmq/ds4_repack.o")
 elseif(DS4_GPU STREQUAL "metal")
     list(APPEND DS4_OBJS "${DS4_DIR}/ds4_metal.o")
 elseif(DS4_GPU STREQUAL "cpu")

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=54b36ed9ba42da31b24f2d1a5feb075c2475dbb1
+# Upstream pin lives below as DS4_VERSION?=b7e9f0091139999b6c070a57590c447c5741da5c
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=54b36ed9ba42da31b24f2d1a5feb075c2475dbb1
+DS4_VERSION?=b7e9f0091139999b6c070a57590c447c5741da5c
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -23,7 +23,9 @@ CMAKE_ARGS ?= -DCMAKE_BUILD_TYPE=Release
 # are shared by every GPU mode, so append them unconditionally below.
 ifeq ($(BUILD_TYPE),cublas)
     CMAKE_ARGS += -DDS4_GPU=cuda
-    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \
+        cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \
+        cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o
 else ifeq ($(UNAME_S),Darwin)
     CMAKE_ARGS += -DDS4_GPU=metal
     DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
@@ -55,7 +57,7 @@ ds4:
 # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA).
 ds4/ds4.o: ds4
 ifeq ($(BUILD_TYPE),cublas)
-	+$(MAKE) -C ds4 ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+	+$(MAKE) -C ds4 $(DS4_OBJ_TARGET)
 else ifeq ($(UNAME_S),Darwin)
 	+$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,30 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=a42b8187caff02c570c28e19e4dc2b1d7f55ed14
+VLLM_CPP_VERSION?=9d1fad3cde0acb95eb0bb0a1025f40a0eb614147
+
+# MLX GEMM provider (darwin/metal only; see the metal branch below for why).
+# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun
+# metal`, i.e. a full Xcode the macOS runners do not have, while the wheel ships
+# include/, lib/libmlx.dylib and the compiled mlx.metallib ready to link.
+#
+# DEFAULT ON, but ONLY because VLLM_CPP_VERSION above is pinned at or past
+# vllm.cpp 89c46aeb, which SHAPE-GATES the provider to prefill. The ordering is
+# load-bearing, not incidental:
+#
+#   pin >= 89c46aeb, MLX on   -> 99.1% of MLX-LM  (gated: prefill only)
+#   pin <  89c46aeb, MLX on   -> ~51%             (ungated: it also takes decode)
+#
+# MLX's steel GEMM wins prefill (537 ms TTFT against 602) and loses decode badly,
+# because the provider pays an mx::eval sync plus an output memcpy per call and
+# decode makes ~112 calls per TOKEN. Ungated it does both; gated it does only the
+# good half. So if this pin is ever moved BACKWARDS, this default must go with it.
+VLLM_CPP_MLX?=on
+MLX_VERSION?=0.29.4
+MLX_VENV?=$(abspath ./mlx-venv)
+# Resolved lazily (recursive `=`, not `:=`): the glob only matches once the venv
+# target has run, and the interpreter version in the path varies per runner.
+MLX_ROOT=$(shell echo $(MLX_VENV)/lib/python*/site-packages/mlx)
 
 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
 # server, examples and tests of the engine are never built here.
@@ -49,6 +72,23 @@ else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF
 else ifeq ($(BUILD_TYPE),metal)
 	CMAKE_ARGS+=-DVLLM_CPP_METAL=ON
+	# The optional MLX GEMM provider. vllm.cpp keeps it OFF by default because it
+	# is a ~19 MB libmlx.dylib plus a ~105 MB mlx.metallib, and upstream's
+	# position is that it must earn that cost by measurement. It does, on the
+	# only hardware this build targets: measured on an Apple M4 against the
+	# native MSL GEMM in the SAME binary (arms toggled by
+	# VT_OP_PROVIDER_DISABLE=mlx), Qwen3-1.7B-bf16 p=512 g=128, it is 1.5x to
+	# 2.2x aggregate throughput and 2x to 3x faster TTFT, at equal peak memory
+	# and bit-identical output on every parity shape. See vllm.cpp
+	# docs/BENCHMARKS.md "MLX GEMM provider A/B on Apple M4".
+	#
+	# MLX delegates the dense GEMM ONLY: kPagedAttention stays vllm.cpp's own
+	# kernel, because MLX has no paged-KV primitive at all.
+	#
+	# Set VLLM_CPP_MLX=off for a Metal build without it (smaller image, slower).
+	ifeq ($(VLLM_CPP_MLX),on)
+		MLX_ENABLED=1
+	endif
 else
 	CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF
 endif
@@ -56,12 +96,6 @@ endif
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Darwin)
 	LIB=libvllm.dylib
-	# Apple Clang diagnoses a pair of constant-folded array bounds in the Metal
-	# build as a GNU extension. Disable that diagnostic for both Objective-C and
-	# C++ because vllm.cpp appends target-local -Werror after these global flags.
-	CMAKE_ARGS+=-DCMAKE_CXX_FLAGS=-Wno-gnu-folding-constant
-	CMAKE_ARGS+=-DCMAKE_OBJC_FLAGS=-Wno-gnu-folding-constant
-	CMAKE_ARGS+=-DCMAKE_OBJCXX_FLAGS=-Wno-gnu-folding-constant
 else
 	LIB=libvllm.so
 endif
@@ -74,10 +108,35 @@ sources/vllm.cpp:
 	git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \
 	git checkout FETCH_HEAD
 
-$(LIB): sources/vllm.cpp
+ifeq ($(MLX_ENABLED),1)
+# A stamp FILE, not a phony target: a phony prerequisite is always "newer" than
+# $(LIB) and would re-link libvllm on every invocation. Keyed on the version so
+# a MLX_VERSION bump reinstalls instead of silently reusing the old wheel.
+MLX_STAMP=$(MLX_VENV)/.mlx-$(MLX_VERSION).stamp
+MLX_CMAKE_ARGS=-DVLLM_CPP_MLX=ON -DMLX_ROOT=$(MLX_ROOT)
+
+$(MLX_STAMP):
+	@if [ ! -x "$(MLX_VENV)/bin/pip" ]; then \
+		python3 -m venv "$(MLX_VENV)" || { echo "vllm-cpp: python3 with venv is required to build the MLX provider; pass VLLM_CPP_MLX=off to build Metal without it" >&2; exit 1; }; \
+	fi
+	"$(MLX_VENV)"/bin/pip install --quiet --disable-pip-version-check "mlx==$(MLX_VERSION)"
+	@# Resolved in the SHELL, not by $(MLX_ROOT): make expands a whole recipe
+	@# before running its first line, so the glob would still be unmatched here.
+	@# Every later use (the cmake args, package.sh) expands after this target has
+	@# completed, where $(MLX_ROOT) does resolve.
+	@root=$$(echo "$(MLX_VENV)"/lib/python*/site-packages/mlx); \
+	test -f "$$root/lib/libmlx.dylib" -a -f "$$root/include/mlx/array.h" || \
+		{ echo "vllm-cpp: mlx==$(MLX_VERSION) did not provide lib/libmlx.dylib + include/mlx/array.h under $$root" >&2; exit 1; }
+	touch $@
+else
+MLX_STAMP=
+MLX_CMAKE_ARGS=
+endif
+
+$(LIB): sources/vllm.cpp $(MLX_STAMP)
 	mkdir -p build && \
 	cd build && \
-	cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \
+	cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \
 	cmake --build . --config Release -j$(JOBS) --target vllm_shared
 	cp -fL build/$(LIB) ./$(LIB)
 
@@ -85,12 +144,12 @@ vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB)
 	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./
 
 package: vllm-cpp
-	bash package.sh
+	MLX_ROOT="$(MLX_ROOT)" bash package.sh
 
 build: package
 
 clean: purge
-	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp
+	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp "$(MLX_VENV)"
 
 purge:
 	rm -rf build

backend/go/vllm-cpp/README.md

@@ -41,5 +41,50 @@ options:
 - max_num_seqs:16
 ```
 
+## Apple Silicon: the MLX GEMM provider (ON by default, gated to prefill)
+
+`BUILD_TYPE=metal` builds vllm.cpp's MLX provider for the dense GEMM
+(`VLLM_CPP_MLX=on`, the default here). It is on because upstream now SHAPE-GATES
+it to prefill; it was briefly off in this branch's history, and that was correct
+at the time for an ungated provider.
+
+The gate matters more than the flag. MLX's steel GEMM wins prefill but loses
+decode, because the provider pays an `mx::eval` synchronisation plus an output
+memcpy on every call and decode makes ~112 calls *per token*. Measured on an
+Apple M4, Qwen3-1.7B-bf16 warm at p=512 g=128:
+
+| configuration | prefill TTFT | warm throughput |
+|---|--:|--:|
+| MLX **gated to prefill** (pin >= 89c46aeb) | **524.5 ms** | **24.37 tok/s, 97.6% of MLX-LM** |
+| MLX ungated (older pins) | 537 ms | 12.7 tok/s |
+| MLX off | 602 ms | 23.9 tok/s, 95.9% |
+
+Ratios are against an MLX-LM baseline measured INTERLEAVED with ours over four
+ABBA blocks (its spread 0.34%, ours 0.12%). An earlier revision of this file
+claimed 99.1%; that used a two-run MLX-LM baseline containing an outlier and
+overstated us by about 1.5 points.
+
+**`VLLM_CPP_VERSION` and this flag are coupled.** Moving the pin back before
+`89c46aeb` while leaving `VLLM_CPP_MLX=on` would take the middle row — roughly
+half throughput. If you roll the pin back, roll the default back with it.
+
+One caveat: MLX's GEMM is not bit-identical to the native kernel, so an MLX build
+produces a different greedy sequence than a non-MLX one. That is a property of the
+provider, not of the gate, and it predates this packaging. Full disposition in
+vllm.cpp `docs/BENCHMARKS.md`.
+
+Build knobs:
+
+- `VLLM_CPP_MLX=off` builds Metal without the provider: ~124 MB smaller, and
+  96.4% of MLX-LM instead of 99.1%.
+- `MLX_VERSION` pins the wheel (default `0.29.4`). MLX is consumed as the
+  prebuilt pip wheel because building it from source needs `xcrun metal`, i.e. a
+  full Xcode the macOS runners do not have.
+
+Packaging vendors `libmlx.dylib`, `mlx.metallib` and MLX's MIT license into
+`package/lib/`, and rewrites `libvllm.dylib`'s rpath to `@loader_path/lib`
+(re-signing it, since `install_name_tool` invalidates the signature). The
+metallib must stay beside `libmlx.dylib`: MLX looks for it there.
+
 Testing: `make test` runs the unit specs; export `VLLM_CPP_MODEL=<model>` (and
 optionally `VLLM_CPP_LIBRARY=<libvllm path>`) to enable the e2e specs.

backend/go/vllm-cpp/backend.go

@@ -109,16 +109,6 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 
 	v.opts = parseOptions(opts)
 
-	// A DFlash draft is a second checkpoint the engine opens by path, and the
-	// engine never downloads one. Resolve it against LocalAI's models directory
-	// now so a repo-id spelling works, and so a missing draft fails here with an
-	// actionable message rather than as an HF-cache miss inside the load.
-	resolvedSpec, err := resolveDraftModelPath(v.opts.speculativeConfig, opts.ModelPath)
-	if err != nil {
-		return err
-	}
-	v.opts.speculativeConfig = resolvedSpec
-
 	mp := defaultModelParams()
 	if v.opts.blockSize > 0 {
 		mp.BlockSize = v.opts.blockSize
@@ -126,62 +116,34 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 	if v.opts.numBlocks > 0 {
 		mp.NumBlocks = v.opts.numBlocks
 	}
-	// Sequence-length precedence, narrowest source last: context_size is the
-	// generic LocalAI knob every backend honours, max_model_len is the
-	// vLLM-specific one, and engine_args.max_model_len is the explicit
-	// vllm-cpp override.
 	if opts.ContextSize > 0 {
 		mp.MaxModelLen = opts.ContextSize
 	}
-	if opts.MaxModelLen > 0 {
-		mp.MaxModelLen = opts.MaxModelLen
-	}
-	if v.opts.maxModelLen > 0 {
-		mp.MaxModelLen = v.opts.maxModelLen
-	}
 	if v.opts.maxNumSeqs > 0 {
 		mp.MaxNumSeqs = v.opts.maxNumSeqs
 	}
-	if v.opts.maxNumBatchedTokens > 0 {
-		mp.MaxNumBatchedTokens = v.opts.maxNumBatchedTokens
-	}
-	mp.EnablePrefixCaching = v.opts.enablePrefixCaching
-	mp.EnableJumpForward = v.opts.enableJumpForward
 
-	// Every string below is borrowed by C for the duration of the load call
-	// only (the library copies what it keeps), so the backing slices just have
-	// to outlive vllmEngineLoad - hence the single KeepAlive after it.
 	modelC := cString(model)
 	mp.ModelPath = uintptr(unsafe.Pointer(&modelC[0])) // #nosec G103 -- borrowed by C for the load call only
-	keep := [][]byte{modelC}
-	setStr := func(dst *uintptr, s string) {
-		if s == "" {
-			return
-		}
-		b := cString(s)
-		keep = append(keep, b)
-		*dst = uintptr(unsafe.Pointer(&b[0])) // #nosec G103 -- borrowed by C for the load call only
+	var toolParserC, reasoningParserC []byte
+	if v.opts.toolParser != "" {
+		toolParserC = cString(v.opts.toolParser)
+		mp.ToolParser = uintptr(unsafe.Pointer(&toolParserC[0])) // #nosec G103 -- borrowed by C for the load call only
+	}
+	if v.opts.reasoningParser != "" {
+		reasoningParserC = cString(v.opts.reasoningParser)
+		mp.ReasoningParser = uintptr(unsafe.Pointer(&reasoningParserC[0])) // #nosec G103 -- borrowed by C for the load call only
 	}
-	setStr(&mp.ToolParser, v.opts.toolParser)
-	setStr(&mp.ReasoningParser, v.opts.reasoningParser)
-	setStr(&mp.SpeculativeConfig, v.opts.speculativeConfig)
-	setStr(&mp.KVTransferConfig, v.opts.kvTransferConfig)
-	setStr(&mp.SchedulingPolicy, v.opts.schedulingPolicy)
-	setStr(&mp.TokenizerConfigPath, v.opts.tokenizerConfigPath)
 
 	xlog.Info("[vllm-cpp] Load", "model", model, "engine", vllmVersion(),
 		"blockSize", mp.BlockSize, "numBlocks", mp.NumBlocks,
-		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs,
-		"maxNumBatchedTokens", mp.MaxNumBatchedTokens,
-		"prefixCaching", triStateName(mp.EnablePrefixCaching),
-		"jumpForward", triStateName(mp.EnableJumpForward),
-		"schedulingPolicy", v.opts.schedulingPolicy,
-		"speculativeConfig", v.opts.speculativeConfig,
-		"kvTransferConfig", v.opts.kvTransferConfig)
+		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs)
 
 	var engine uintptr
 	rc := vllmEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params
-	runtime.KeepAlive(keep)
+	runtime.KeepAlive(modelC)
+	runtime.KeepAlive(toolParserC)
+	runtime.KeepAlive(reasoningParserC)
 	if rc != vllmOK {
 		return fmt.Errorf("vllm-cpp: engine load failed: %s", vllmLastError())
 	}

backend/go/vllm-cpp/govllmcpp.go

@@ -1,6 +1,6 @@
 package main
 
-// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10).
+// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v2).
 //
 // The structs below are hand-mirrored PODs of the C declarations, with
 // explicit padding so the Go layout matches the C layout on linux/darwin
@@ -18,56 +18,23 @@ import (
 )
 
 // abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h).
-const abiVersion = 10
-
-// The ABI's tri-state toggles (enable_prefix_caching ABI v7,
-// enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
-// "defer" - to the model capability for prefix caching, to the environment for
-// jump forward. Only 2 is an explicit off.
-const (
-	triStateDefer int32 = 0
-	triStateOn    int32 = 1
-	triStateOff   int32 = 2
-)
-
-// triStateName renders a tri-state for the load log line, where "0" would
-// otherwise read as "off" rather than "whatever the default resolves to".
-func triStateName(state int32) string {
-	switch state {
-	case triStateOn:
-		return "on"
-	case triStateOff:
-		return "off"
-	default:
-		return "model-default"
-	}
-}
+const abiVersion = 5
 
 // vllm_status (vllm.h).
 const (
 	vllmOK = 0
 )
 
-// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the
-// interior needs no padding on LP64, but the struct is 8-aligned (it holds
-// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets
-// and total size are asserted in vllmcpp_test.go.
+// cModelParams mirrors vllm_model_params.
 type cModelParams struct {
 	ModelPath           uintptr // const char*
-	TokenizerConfigPath uintptr // const char*; NULL = <model_dir>/... (ABI v9)
+	TokenizerConfigPath uintptr // const char*
 	BlockSize           int32
 	NumBlocks           int32
 	MaxModelLen         int32
 	MaxNumSeqs          int32
 	ToolParser          uintptr // const char*; NULL = auto-detect (ABI v4)
 	ReasoningParser     uintptr // const char*; NULL = auto-detect (ABI v5)
-	SpeculativeConfig   uintptr // const char* JSON; NULL = no speculation (ABI v6)
-	EnablePrefixCaching int32   // tri-state 0/1/2 (ABI v7)
-	MaxNumBatchedTokens int32   // <= 0 = per-arch default (ABI v9)
-	SchedulingPolicy    uintptr // const char*; NULL = "fcfs" (ABI v9)
-	KVTransferConfig    uintptr // const char* JSON; NULL = no connector (ABI v9)
-	EnableJumpForward   int32   // tri-state 0/1/2 (ABI v10)
-	_                   [4]byte // trailing pad to the struct's 8-byte alignment
 }
 
 // cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields
@@ -98,12 +65,6 @@ type cSamplingParams struct {
 	StructuredGrammar    uintptr // const char*
 	StructuredJSONObject int32
 	_                    [4]byte
-	// ABI v8 tail. LocalAI installs no custom logits processor, but the fields
-	// MUST be mirrored: the C side reads them off the pointer we hand it, so a
-	// Go struct that stopped at StructuredJSONObject would have the engine read
-	// 16 bytes past our allocation and call whatever garbage sat there.
-	LogitsProcessor         uintptr // vllm_logits_processor; NULL = none
-	LogitsProcessorUserData uintptr // void*
 }
 
 // cCompletion mirrors vllm_completion.

backend/go/vllm-cpp/options.go

@@ -1,80 +1,30 @@
 package main
 
-// Load-time engine configuration, from two config surfaces:
-//
-//   - `engine_args:` (ModelOptions.EngineArgs, a JSON object) is the canonical
-//     one. Keys are spelled exactly as vLLM's own CLI flags, so a config written
-//     against vLLM works verbatim here - `speculative_config` and
-//     `kv_transfer_config` in particular take the same JSON documents vLLM's
-//     --speculative-config / --kv-transfer-config accept, and are handed to the
-//     engine unparsed.
-//   - `options:` (the free-form "key:value" list) is the older surface this
-//     backend shipped with. It is still honoured so existing configs keep
-//     working; engine_args wins on any key set in both.
-//
-// Anything unrecognised is ignored rather than fatal: the engine validates the
-// documents it is given and reports a precise error at load, and a config that
-// also carries knobs for a different backend must not fail the load here.
+// Engine-sizing knobs carried through the model config's free-form
+// `options:` list ("key:value" entries), mirroring how the other in-house
+// backends pass engine-specific settings that have no proto field.
 
 import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path"
-	"path/filepath"
 	"strconv"
 	"strings"
 
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
-	"github.com/mudler/xlog"
 )
 
 type loadOptions struct {
 	blockSize  int32 // KV block size (tokens/block); engine default 32.
 	numBlocks  int32 // KV blocks to allocate; engine default 256.
 	maxNumSeqs int32 // max concurrent sequences; engine default 8.
-	// Max sequence length. Also settable through the model config's
-	// context_size / max_model_len; see Load for the precedence.
-	maxModelLen int32
-	// Per-step chunked-prefill token budget (ABI v9). 0 = the engine's
-	// bounded per-arch default.
-	maxNumBatchedTokens int32
-	// Automatic prefix caching tri-state (ABI v7): 0 = the model-capability
-	// default, 1 = force on, 2 = force off.
-	enablePrefixCaching int32
-	// Jump-forward decoding tri-state (ABI v10), SGLang's grammar-speed subset:
-	// 0 = defer to the environment (VT_ENABLE_JUMP_FORWARD, default off),
-	// 1 = force on, 2 = force off.
-	enableJumpForward int32
-	// Scheduler admission policy (ABI v9): "" = fcfs, else fcfs|priority|lpm.
-	schedulingPolicy string
 	// Engine-side parser selection (ABI v4/v5). Empty = the engine
 	// auto-detects from the chat template; "none" disables the reasoning
 	// split; unknown names fail the first chat call.
 	toolParser      string
 	reasoningParser string
-	// Speculative decoding (ABI v6), as vLLM's --speculative-config JSON:
-	// {"method":"mtp"|"dflash"|"ngram", ...}. Empty = no speculation.
-	speculativeConfig string
-	// External KV connector / LMCache (ABI v9), as vLLM's --kv-transfer-config
-	// JSON. Empty = no connector.
-	kvTransferConfig string
-	// Override for the tokenizer_config.json the chat template is read from
-	// (ABI v9). Empty = <model_dir>/tokenizer_config.json.
-	tokenizerConfigPath string
 }
 
 func parseOptions(opts *pb.ModelOptions) loadOptions {
 	lo := loadOptions{}
-	applyOptionsList(&lo, opts.GetOptions())
-	applyEngineArgs(&lo, opts.GetEngineArgs())
-	return lo
-}
-
-// applyOptionsList reads the legacy free-form "key:value" list. strings.Cut
-// splits on the FIRST colon only, so a JSON object value survives intact.
-func applyOptionsList(lo *loadOptions, options []string) {
-	for _, o := range options {
+	for _, o := range opts.GetOptions() {
 		k, v, found := strings.Cut(o, ":")
 		if !found {
 			continue
@@ -86,211 +36,13 @@ func applyOptionsList(lo *loadOptions, options []string) {
 			lo.numBlocks = parseInt32(v, lo.numBlocks)
 		case "max_num_seqs":
 			lo.maxNumSeqs = parseInt32(v, lo.maxNumSeqs)
-		case "max_num_batched_tokens":
-			lo.maxNumBatchedTokens = parseInt32(v, lo.maxNumBatchedTokens)
-		case "max_model_len":
-			lo.maxModelLen = parseInt32(v, lo.maxModelLen)
-		case "scheduling_policy", "schedule_policy":
-			lo.schedulingPolicy = strings.TrimSpace(v)
-		case "tool_parser", "tool_call_parser":
+		case "tool_parser":
 			lo.toolParser = strings.TrimSpace(v)
 		case "reasoning_parser":
 			lo.reasoningParser = strings.TrimSpace(v)
-		case "speculative_config":
-			lo.speculativeConfig = strings.TrimSpace(v)
-		case "kv_transfer_config":
-			lo.kvTransferConfig = strings.TrimSpace(v)
-		case "tokenizer_config", "tokenizer_config_path":
-			lo.tokenizerConfigPath = strings.TrimSpace(v)
-		case "enable_prefix_caching", "enable_radix_attention":
-			if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
-				lo.enablePrefixCaching = boolTriState(b)
-			}
-		case "enable_jump_forward":
-			if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
-				lo.enableJumpForward = boolTriState(b)
-			}
 		}
 	}
-}
-
-// applyEngineArgs overlays the `engine_args:` JSON object. A document that does
-// not parse is logged and skipped: engine_args is shared with the other engines
-// (the vLLM and SGLang backends read the same field), so a stray key must not
-// take the model down.
-func applyEngineArgs(lo *loadOptions, engineArgs string) {
-	if strings.TrimSpace(engineArgs) == "" {
-		return
-	}
-	var args map[string]any
-	if err := json.Unmarshal([]byte(engineArgs), &args); err != nil {
-		xlog.Warn("[vllm-cpp] ignoring unparseable engine_args", "error", err)
-		return
-	}
-	for k, v := range args {
-		switch k {
-		case "block_size":
-			lo.blockSize = jsonInt32(v, lo.blockSize)
-		case "num_blocks":
-			lo.numBlocks = jsonInt32(v, lo.numBlocks)
-		case "max_num_seqs":
-			lo.maxNumSeqs = jsonInt32(v, lo.maxNumSeqs)
-		case "max_num_batched_tokens":
-			lo.maxNumBatchedTokens = jsonInt32(v, lo.maxNumBatchedTokens)
-		case "max_model_len":
-			lo.maxModelLen = jsonInt32(v, lo.maxModelLen)
-		case "scheduling_policy", "schedule_policy":
-			lo.schedulingPolicy = jsonString(v, lo.schedulingPolicy)
-		case "tool_parser", "tool_call_parser":
-			lo.toolParser = jsonString(v, lo.toolParser)
-		case "reasoning_parser":
-			lo.reasoningParser = jsonString(v, lo.reasoningParser)
-		case "tokenizer_config", "tokenizer_config_path":
-			lo.tokenizerConfigPath = jsonString(v, lo.tokenizerConfigPath)
-		case "speculative_config":
-			lo.speculativeConfig = jsonDocument(v, lo.speculativeConfig, k)
-		case "kv_transfer_config":
-			lo.kvTransferConfig = jsonDocument(v, lo.kvTransferConfig, k)
-		case "enable_prefix_caching", "enable_radix_attention":
-			if b, ok := v.(bool); ok {
-				lo.enablePrefixCaching = boolTriState(b)
-			}
-		case "enable_jump_forward":
-			if b, ok := v.(bool); ok {
-				lo.enableJumpForward = boolTriState(b)
-			}
-		default:
-			xlog.Debug("[vllm-cpp] ignoring unknown engine_args key", "key", k)
-		}
-	}
-}
-
-// boolTriState maps a YAML/JSON boolean onto the ABI's tri-state encoding. An
-// explicit `false` must reach the engine as force-OFF (2), NOT as the 0 that
-// means "defer". The difference is real in both directions: prefix caching
-// defaults ON for dense archs and OFF for hybrid ones, and jump forward defers
-// to VT_ENABLE_JUMP_FORWARD.
-func boolTriState(on bool) int32 {
-	if on {
-		return triStateOn
-	}
-	return triStateOff
-}
-
-// jsonDocument normalises an object-valued engine_args entry to a JSON string
-// for the C ABI. YAML nesting arrives as a map (the natural spelling); a
-// pre-encoded JSON string is accepted too, since a config round-tripped through
-// a flat store may carry it that way.
-func jsonDocument(v any, fallback string, key string) string {
-	switch t := v.(type) {
-	case string:
-		if strings.TrimSpace(t) == "" {
-			return fallback
-		}
-		return t
-	default:
-		buf, err := json.Marshal(t)
-		if err != nil {
-			xlog.Warn("[vllm-cpp] ignoring unencodable engine_args value", "key", key, "error", err)
-			return fallback
-		}
-		return string(buf)
-	}
-}
-
-func jsonString(v any, fallback string) string {
-	s, ok := v.(string)
-	if !ok {
-		return fallback
-	}
-	return strings.TrimSpace(s)
-}
-
-// jsonInt32 accepts the float64 a JSON number decodes to, plus the string
-// spelling a YAML config may produce. Non-positive values keep the fallback:
-// every knob this covers uses "<= 0 means the engine default".
-func jsonInt32(v any, fallback int32) int32 {
-	switch t := v.(type) {
-	case float64:
-		if t <= 0 || t > 1<<31-1 {
-			return fallback
-		}
-		return int32(t)
-	case string:
-		return parseInt32(t, fallback)
-	default:
-		return fallback
-	}
-}
-
-// resolveDraftModelPath rewrites a DFlash draft reference into an absolute path
-// the engine can actually open.
-//
-// The engine resolves `speculative_config.model` against a directory containing
-// config.json, or against ~/.cache/huggingface/hub/models--<org>--<repo>/
-// snapshots/* - and it NEVER downloads. LocalAI keeps models in its own
-// directory, so a bare HF repo id (the spelling the vLLM docs teach) misses the
-// HF cache and dies deep in the load with "draft checkpoint not found", which
-// reads like a broken checkpoint rather than a missing download.
-//
-// So: try the reference as given, then the last path segment under the models
-// dir (`z-lab/Qwen3.6-27B-DFlash` -> `<models>/Qwen3.6-27B-DFlash`, which is
-// what LocalAI's own downloader produces), then the whole reference under the
-// models dir. If none exist, fail HERE with a message naming both what was
-// asked for and where we looked.
-//
-// mtp and ngram carry no separate draft checkpoint, so they pass through. A
-// document that does not parse also passes through: the engine owns config
-// validation and produces the better error.
-func resolveDraftModelPath(speculativeConfig, modelsDir string) (string, error) {
-	if strings.TrimSpace(speculativeConfig) == "" {
-		return speculativeConfig, nil
-	}
-	var spec map[string]any
-	if err := json.Unmarshal([]byte(speculativeConfig), &spec); err != nil {
-		return speculativeConfig, nil
-	}
-	if method, _ := spec["method"].(string); !strings.EqualFold(method, "dflash") {
-		return speculativeConfig, nil
-	}
-
-	ref, _ := spec["model"].(string)
-	ref = strings.TrimSpace(ref)
-	if ref == "" {
-		return "", fmt.Errorf(
-			"vllm-cpp: speculative_config method %q requires a \"model\" key naming the draft checkpoint", "dflash")
-	}
-
-	candidates := []string{ref}
-	if modelsDir != "" {
-		if base := path.Base(filepath.ToSlash(ref)); base != "" && base != "." && base != "/" {
-			candidates = append(candidates, filepath.Join(modelsDir, base))
-		}
-		candidates = append(candidates, filepath.Join(modelsDir, filepath.FromSlash(ref)))
-	}
-
-	for _, c := range candidates {
-		if _, err := os.Stat(filepath.Join(c, "config.json")); err != nil {
-			continue
-		}
-		abs, err := filepath.Abs(c)
-		if err != nil {
-			abs = c
-		}
-		spec["model"] = abs
-		out, err := json.Marshal(spec)
-		if err != nil {
-			return "", fmt.Errorf("vllm-cpp: re-encoding speculative_config: %w", err)
-		}
-		xlog.Info("[vllm-cpp] resolved DFlash draft checkpoint", "reference", ref, "path", abs)
-		return string(out), nil
-	}
-
-	return "", fmt.Errorf(
-		"vllm-cpp: DFlash draft checkpoint %q not found (looked in: %s). "+
-			"The engine does not download drafts - install the draft model into LocalAI first, "+
-			"or set speculative_config.model to an absolute path to a directory containing config.json",
-		ref, strings.Join(candidates, ", "))
+	return lo
 }
 
 func parseInt32(s string, fallback int32) int32 {

backend/go/vllm-cpp/package.sh

@@ -43,6 +43,50 @@ elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
     cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
 elif [ $(uname -s) = "Darwin" ]; then
     echo "Detected Darwin"
+    # Vendor the optional MLX GEMM provider, when libvllm was built against it.
+    # Three facts drive every line below, each verified on an Apple M4 before it
+    # was written:
+    #   1. libvllm.dylib carries an LC_LOAD_DYLIB on @rpath/libmlx.dylib, and its
+    #      build-time LC_RPATH points inside the build venv. That path does not
+    #      exist on a user's machine, so it must become @loader_path/lib.
+    #   2. MLX finds its ~100 MB mlx.metallib beside its OWN dylib, so the two
+    #      files have to land in the same directory or every Metal op dies with
+    #      "Failed to load the default metallib".
+    #   3. install_name_tool invalidates the code signature, and macOS refuses to
+    #      load an arm64 image whose signature does not match, so the patched
+    #      library must be re-signed ad-hoc afterwards.
+    if otool -L "$CURDIR/package/libvllm.dylib" 2>/dev/null | grep -q "libmlx.dylib"; then
+        MLX_LIB_DIR="${MLX_ROOT}/lib"
+        if [ ! -f "$MLX_LIB_DIR/libmlx.dylib" ] || [ ! -f "$MLX_LIB_DIR/mlx.metallib" ]; then
+            echo "Error: libvllm.dylib links libmlx.dylib but $MLX_LIB_DIR is missing libmlx.dylib/mlx.metallib" >&2
+            exit 1
+        fi
+        echo "Vendoring the MLX GEMM provider from $MLX_LIB_DIR"
+        cp -fLv "$MLX_LIB_DIR/libmlx.dylib" "$CURDIR/package/lib/"
+        cp -fLv "$MLX_LIB_DIR/mlx.metallib" "$CURDIR/package/lib/"
+        # MLX is MIT and we redistribute its binaries, so its license ships with
+        # them. mlx-metal is the wheel carrying the dylib and the metallib.
+        MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx_metal-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
+        if [ -z "$MLX_LICENSE" ]; then
+            MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
+        fi
+        if [ -z "$MLX_LICENSE" ]; then
+            echo "Error: could not find the MLX LICENSE to redistribute alongside libmlx.dylib" >&2
+            exit 1
+        fi
+        cp -fLv "$MLX_LICENSE" "$CURDIR/package/lib/LICENSE.mlx"
+        # Drop every build-tree rpath, then point at the packaged copy.
+        otool -l "$CURDIR/package/libvllm.dylib" | awk '/LC_RPATH/{f=1;next} f&&/ path /{print $2;f=0}' | while read -r rp; do
+            install_name_tool -delete_rpath "$rp" "$CURDIR/package/libvllm.dylib" 2>/dev/null || true
+        done
+        install_name_tool -add_rpath "@loader_path/lib" "$CURDIR/package/libvllm.dylib"
+        codesign -f -s - "$CURDIR/package/libvllm.dylib"
+        # A broken rpath must fail the BUILD, not the user's first inference.
+        if ! otool -l "$CURDIR/package/libvllm.dylib" | grep -q "@loader_path/lib"; then
+            echo "Error: libvllm.dylib did not get the @loader_path/lib rpath" >&2
+            exit 1
+        fi
+    fi
 else
     echo "Error: Could not detect architecture"
     exit 1

backend/go/vllm-cpp/vllmcpp_test.go

@@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) {
 	RunSpecs(t, "vllm-cpp suite")
 }
 
-// The Go POD mirrors must match the C struct layout of vllm.h (ABI v9)
+// The Go POD mirrors must match the C struct layout of vllm.h (ABI v2)
 // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
 // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
 var _ = Describe("C ABI struct mirrors", func() {
@@ -30,18 +30,10 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.MaxNumSeqs)).To(Equal(uintptr(28)))
 		Expect(unsafe.Offsetof(p.ToolParser)).To(Equal(uintptr(32)))
 		Expect(unsafe.Offsetof(p.ReasoningParser)).To(Equal(uintptr(40)))
-		Expect(unsafe.Offsetof(p.SpeculativeConfig)).To(Equal(uintptr(48)))
-		Expect(unsafe.Offsetof(p.EnablePrefixCaching)).To(Equal(uintptr(56)))
-		Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60)))
-		Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64)))
-		Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72)))
-		Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80)))
-		// 88, not 84: the struct is 8-aligned (it holds pointers), so the
-		// trailing int32 is padded out. Go pads identically.
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88)))
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(48)))
 	})
 
-	It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() {
+	It("cSamplingParams matches vllm_sampling_params (ABI v2)", func() {
 		var p cSamplingParams
 		Expect(unsafe.Offsetof(p.Temperature)).To(Equal(uintptr(0)))
 		Expect(unsafe.Offsetof(p.TopP)).To(Equal(uintptr(4)))
@@ -63,9 +55,7 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.NStructuredChoice)).To(Equal(uintptr(96)))
 		Expect(unsafe.Offsetof(p.StructuredGrammar)).To(Equal(uintptr(104)))
 		Expect(unsafe.Offsetof(p.StructuredJSONObject)).To(Equal(uintptr(112)))
-		Expect(unsafe.Offsetof(p.LogitsProcessor)).To(Equal(uintptr(120)))
-		Expect(unsafe.Offsetof(p.LogitsProcessorUserData)).To(Equal(uintptr(128)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136)))
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120)))
 	})
 
 	It("cCompletion matches vllm_completion", func() {
@@ -78,23 +68,6 @@ var _ = Describe("C ABI struct mirrors", func() {
 	})
 })
 
-// Pin/mirror skew is the failure mode this backend is most exposed to: the Go
-// PODs above are hand-written against one VLLM_ABI_VERSION, and the Makefile
-// pins the vllm.cpp commit that produces it. This spec catches drift without
-// needing model weights - set VLLM_CPP_LIBRARY to a built libvllm and it binds
-// every symbol and compares the library's reported ABI against the mirrors'.
-var _ = Describe("real library ABI handshake", func() {
-	It("binds every symbol and reports the ABI the mirrors were written against", func() {
-		lib := os.Getenv("VLLM_CPP_LIBRARY")
-		if lib == "" {
-			Skip("VLLM_CPP_LIBRARY not set; skipping the real-library handshake")
-		}
-		Expect(registerLib(lib)).To(Succeed())
-		Expect(vllmABIVersion()).To(Equal(int32(abiVersion)))
-		Expect(vllmVersion()).NotTo(BeEmpty())
-	})
-})
-
 var _ = Describe("parseOptions", func() {
 	It("extracts the engine sizing knobs", func() {
 		lo := parseOptions(&pb.ModelOptions{Options: []string{
@@ -110,129 +83,6 @@ var _ = Describe("parseOptions", func() {
 		}})
 		Expect(lo).To(Equal(loadOptions{}))
 	})
-
-	It("carries a speculative_config JSON value through the legacy options list", func() {
-		// strings.Cut splits on the FIRST colon only, so a JSON object value
-		// survives the "key:value" spelling intact.
-		lo := parseOptions(&pb.ModelOptions{Options: []string{
-			`speculative_config:{"method":"mtp","num_speculative_tokens":1}`,
-		}})
-		Expect(lo.speculativeConfig).To(Equal(`{"method":"mtp","num_speculative_tokens":1}`))
-	})
-})
-
-var _ = Describe("engine_args", func() {
-	It("maps every load knob onto the C model params", func() {
-		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
-			"block_size": 64,
-			"num_blocks": 1024,
-			"max_model_len": 16384,
-			"max_num_seqs": 32,
-			"max_num_batched_tokens": 8192,
-			"enable_prefix_caching": true,
-			"scheduling_policy": "lpm",
-			"tool_parser": "qwen3",
-			"reasoning_parser": "deepseek_r1",
-			"tokenizer_config": "/models/tok/tokenizer_config.json"
-		}`})
-		Expect(lo.blockSize).To(Equal(int32(64)))
-		Expect(lo.numBlocks).To(Equal(int32(1024)))
-		Expect(lo.maxModelLen).To(Equal(int32(16384)))
-		Expect(lo.maxNumSeqs).To(Equal(int32(32)))
-		Expect(lo.maxNumBatchedTokens).To(Equal(int32(8192)))
-		Expect(lo.enablePrefixCaching).To(Equal(int32(1)))
-		Expect(lo.schedulingPolicy).To(Equal("lpm"))
-		Expect(lo.toolParser).To(Equal("qwen3"))
-		Expect(lo.reasoningParser).To(Equal("deepseek_r1"))
-		Expect(lo.tokenizerConfigPath).To(Equal("/models/tok/tokenizer_config.json"))
-	})
-
-	It("re-marshals a nested speculative_config object to JSON for the engine", func() {
-		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
-			"speculative_config": {"method": "mtp", "num_speculative_tokens": 1}
-		}`})
-		Expect(lo.speculativeConfig).To(MatchJSON(`{"method":"mtp","num_speculative_tokens":1}`))
-	})
-
-	It("re-marshals a nested kv_transfer_config object (LMCache) to JSON", func() {
-		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
-			"kv_transfer_config": {
-				"kv_connector": "LMCacheConnector",
-				"kv_role": "kv_both",
-				"kv_connector_extra_config": {"host": "127.0.0.1", "port": 65432}
-			}
-		}`})
-		Expect(lo.kvTransferConfig).To(MatchJSON(`{
-			"kv_connector":"LMCacheConnector",
-			"kv_role":"kv_both",
-			"kv_connector_extra_config":{"host":"127.0.0.1","port":65432}
-		}`))
-	})
-
-	It("accepts a pre-encoded JSON string for the object-valued knobs", func() {
-		// A config written by hand (or round-tripped through a flat store) may
-		// carry the object as a string; both spellings reach the engine the same.
-		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
-			"speculative_config": "{\"method\":\"ngram\",\"num_speculative_tokens\":4}"
-		}`})
-		Expect(lo.speculativeConfig).To(MatchJSON(`{"method":"ngram","num_speculative_tokens":4}`))
-	})
-
-	It("maps enable_prefix_caching false onto the force-OFF tri-state", func() {
-		// The C ABI tri-state is 0=model default, 1=on, 2=off, so an explicit
-		// `false` must NOT collapse to the 0 that means "let the model decide".
-		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_prefix_caching": false}`})
-		Expect(lo.enablePrefixCaching).To(Equal(int32(2)))
-	})
-
-	It("leaves the prefix-caching tri-state at the model default when unset", func() {
-		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`})
-		Expect(lo.enablePrefixCaching).To(Equal(int32(0)))
-	})
-
-	It("accepts the radix-attention alias upstream documents for prefix caching", func() {
-		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_radix_attention": true}`})
-		Expect(lo.enablePrefixCaching).To(Equal(int32(1)))
-	})
-
-	It("maps enable_jump_forward onto its own tri-state", func() {
-		// ABI v10. Same tri-state shape as prefix caching, and the same trap:
-		// an explicit false must be force-OFF (2), not the 0 that defers to the
-		// environment.
-		on := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": true}`})
-		Expect(on.enableJumpForward).To(Equal(int32(1)))
-		off := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": false}`})
-		Expect(off.enableJumpForward).To(Equal(int32(2)))
-		unset := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`})
-		Expect(unset.enableJumpForward).To(Equal(int32(0)))
-	})
-
-	It("reads enable_jump_forward from the legacy options list too", func() {
-		lo := parseOptions(&pb.ModelOptions{Options: []string{"enable_jump_forward:true"}})
-		Expect(lo.enableJumpForward).To(Equal(int32(1)))
-	})
-
-	It("lets engine_args override the legacy options list", func() {
-		lo := parseOptions(&pb.ModelOptions{
-			Options:    []string{"max_num_seqs:8", "block_size:16"},
-			EngineArgs: `{"max_num_seqs": 64}`,
-		})
-		Expect(lo.maxNumSeqs).To(Equal(int32(64))) // engine_args wins
-		Expect(lo.blockSize).To(Equal(int32(16)))  // untouched keys survive
-	})
-
-	It("ignores malformed engine_args rather than failing the load", func() {
-		lo := parseOptions(&pb.ModelOptions{
-			Options:    []string{"max_num_seqs:8"},
-			EngineArgs: `{not json`,
-		})
-		Expect(lo.maxNumSeqs).To(Equal(int32(8)))
-	})
-
-	It("ignores unknown keys", func() {
-		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"gpu_memory_utilization": 0.9}`})
-		Expect(lo).To(Equal(loadOptions{}))
-	})
 })
 
 var _ = Describe("samplingFromPredict", func() {
@@ -285,91 +135,6 @@ var _ = Describe("samplingFromPredict", func() {
 	})
 })
 
-// The engine resolves speculative_config.model against a local directory or
-// ~/.cache/huggingface/hub ONLY - it never downloads. LocalAI keeps models in
-// its own directory, so a bare repo id would miss the HF cache and fail deep in
-// the load with a confusing "draft checkpoint not found". Resolve it here.
-var _ = Describe("resolveDraftModelPath", func() {
-	var modelsDir string
-
-	BeforeEach(func() {
-		modelsDir = GinkgoT().TempDir()
-	})
-
-	// draftDir creates a plausible draft checkpoint under models/.
-	draftDir := func(name string) string {
-		d := filepath.Join(modelsDir, name)
-		Expect(os.MkdirAll(d, 0o750)).To(Succeed())
-		Expect(os.WriteFile(filepath.Join(d, "config.json"), []byte("{}"), 0o600)).To(Succeed())
-		return d
-	}
-
-	It("rewrites a repo id to the matching directory in the models dir", func() {
-		want := draftDir("Qwen3.6-27B-DFlash")
-		spec := `{"method":"dflash","model":"z-lab/Qwen3.6-27B-DFlash"}`
-		out, err := resolveDraftModelPath(spec, modelsDir)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(out).To(MatchJSON(`{"method":"dflash","model":"` + want + `"}`))
-	})
-
-	It("rewrites a models-dir-relative path", func() {
-		want := draftDir("drafts__dflash")
-		spec := `{"method":"dflash","model":"drafts__dflash"}`
-		out, err := resolveDraftModelPath(spec, modelsDir)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(out).To(ContainSubstring(want))
-	})
-
-	It("leaves an absolute path that already resolves alone", func() {
-		abs := draftDir("elsewhere")
-		spec := `{"method":"dflash","model":"` + abs + `"}`
-		out, err := resolveDraftModelPath(spec, modelsDir)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(out).To(MatchJSON(spec))
-	})
-
-	It("fails with an actionable error when the draft is nowhere on disk", func() {
-		// Silently passing the repo id through would surface as an HF-cache
-		// miss inside the engine, which reads as "your model is broken".
-		spec := `{"method":"dflash","model":"z-lab/Not-Downloaded"}`
-		_, err := resolveDraftModelPath(spec, modelsDir)
-		Expect(err).To(HaveOccurred())
-		Expect(err.Error()).To(ContainSubstring("z-lab/Not-Downloaded"))
-		Expect(err.Error()).To(ContainSubstring(modelsDir))
-	})
-
-	It("requires a model key for dflash", func() {
-		_, err := resolveDraftModelPath(`{"method":"dflash"}`, modelsDir)
-		Expect(err).To(HaveOccurred())
-		Expect(err.Error()).To(ContainSubstring("model"))
-	})
-
-	It("leaves mtp and ngram configs untouched", func() {
-		// Neither has a separate draft checkpoint to resolve.
-		for _, spec := range []string{
-			`{"method":"mtp"}`,
-			`{"method":"ngram","num_speculative_tokens":4}`,
-		} {
-			out, err := resolveDraftModelPath(spec, modelsDir)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(out).To(MatchJSON(spec))
-		}
-	})
-
-	It("passes a malformed document through for the engine to reject", func() {
-		// The engine owns config validation and produces the better message.
-		out, err := resolveDraftModelPath(`{not json`, modelsDir)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(out).To(Equal(`{not json`))
-	})
-
-	It("is a no-op on an empty config", func() {
-		out, err := resolveDraftModelPath("", modelsDir)
-		Expect(err).ToNot(HaveOccurred())
-		Expect(out).To(BeEmpty())
-	})
-})
-
 var _ = Describe("validModelPath", func() {
 	It("accepts a .gguf file", func() {
 		dir := GinkgoT().TempDir()

core/config/vllm_spec.go

@@ -1,117 +0,0 @@
-package config
-
-// Speculative-decoding auto-defaults for the vllm-cpp backend, the safetensors
-// counterpart of the GGUF/llama.cpp hook in mtp.go.
-//
-// The two engines detect and spell the same feature differently. llama.cpp
-// reads `<arch>.nextn_predict_layers` out of the GGUF header and takes
-// `spec_type:draft-mtp` in `options:`; vllm.cpp reads `mtp_num_hidden_layers`
-// out of the checkpoint's config.json and takes vLLM's own
-// `--speculative-config` JSON, which LocalAI carries in `engine_args`. The
-// engine resolves the draft depth and the default k itself, so the config only
-// has to name the method.
-
-import (
-	"encoding/json"
-
-	"github.com/mudler/xlog"
-)
-
-// hfSpecConfig is the subset of a HuggingFace config.json that decides whether
-// speculative decoding can be auto-enabled.
-type hfSpecConfig struct {
-	ModelType string `json:"model_type"`
-	// MtpNumHiddenLayers is the MTP head depth (upstream speculative.py reads
-	// it as n_predict for the qwen3_5 / qwen3_5_moe families).
-	MtpNumHiddenLayers uint32 `json:"mtp_num_hidden_layers"`
-	// DFlashConfig marks a z-lab DFlash DRAFT checkpoint (mask_token_id +
-	// target_layer_ids). Its presence means this repo is a draft, not a
-	// servable target.
-	DFlashConfig json.RawMessage `json:"dflash_config"`
-	// TextConfig is where multimodal checkpoints nest the language-model
-	// config, and therefore the MTP depth.
-	TextConfig *hfSpecConfig `json:"text_config"`
-}
-
-// parseHFSpecConfig decodes the speculative-relevant subset of a config.json.
-// A document that does not parse yields nothing rather than an error: detection
-// is best-effort and must never break an import.
-func parseHFSpecConfig(configJSON []byte) (hfSpecConfig, bool) {
-	if len(configJSON) == 0 {
-		return hfSpecConfig{}, false
-	}
-	var c hfSpecConfig
-	if err := json.Unmarshal(configJSON, &c); err != nil {
-		xlog.Debug("[vllm-spec] config.json did not parse; skipping detection", "error", err)
-		return hfSpecConfig{}, false
-	}
-	return c, true
-}
-
-// IsDFlashDraftConfig reports whether a HuggingFace config.json describes a
-// DFlash DRAFT checkpoint. Unlike MTP - whose head ships inside the target
-// checkpoint's `mtp.*` tensors - a DFlash draft is its own repo that can only
-// run paired with a target it verifies against, so it must never be configured
-// as a standalone model.
-func IsDFlashDraftConfig(configJSON []byte) bool {
-	c, ok := parseHFSpecConfig(configJSON)
-	if !ok {
-		return false
-	}
-	return len(c.DFlashConfig) > 0 ||
-		(c.TextConfig != nil && len(c.TextConfig.DFlashConfig) > 0)
-}
-
-// HasSafetensorsMTPHead reports whether a HuggingFace config.json declares a
-// self-speculating Multi-Token Prediction head, returning its depth. The depth
-// is informational: vllm.cpp resolves n_predict and the default
-// num_speculative_tokens from the checkpoint itself.
-//
-// DFlash drafts are excluded for the same reason `gemma4-assistant` GGUFs are
-// excluded from the llama.cpp hook: they carry head metadata but cannot
-// self-speculate.
-//
-// NOTE this is a safetensors-only signal. vllm.cpp rejects an MTP config over a
-// GGUF source, because the `mtp.*` draft tensors only exist in the safetensors
-// checkpoint - so the GGUF import path must not use this.
-func HasSafetensorsMTPHead(configJSON []byte) (uint32, bool) {
-	c, ok := parseHFSpecConfig(configJSON)
-	if !ok {
-		return 0, false
-	}
-	if IsDFlashDraftConfig(configJSON) {
-		return 0, false
-	}
-	n := c.MtpNumHiddenLayers
-	if n == 0 && c.TextConfig != nil {
-		n = c.TextConfig.MtpNumHiddenLayers
-	}
-	return n, n > 0
-}
-
-// ApplyVLLMSpeculativeDefaults enables MTP speculative decoding in cfg's
-// engine_args when nothing is configured there yet. It is a no-op when the user
-// already set a speculative_config, so an explicit choice (a different method,
-// an explicit k, a DFlash draft) is never clobbered.
-//
-// `layers` is the detected head depth and is only used for the diagnostic log
-// line - the engine derives the real k from the checkpoint.
-func ApplyVLLMSpeculativeDefaults(cfg *ModelConfig, layers uint32) {
-	if cfg == nil {
-		return
-	}
-	if _, set := cfg.EngineArgs["speculative_config"]; set {
-		xlog.Debug("[vllm-spec] MTP head detected but speculative_config already configured; leaving user choice intact",
-			"name", cfg.Name, "mtp_num_hidden_layers", layers)
-		return
-	}
-	if cfg.EngineArgs == nil {
-		cfg.EngineArgs = map[string]any{}
-	}
-	// Only the method: vllm.cpp defaults num_speculative_tokens to the
-	// checkpoint's own n_predict (speculative.py:865-875), which is the right
-	// value far more reliably than anything guessable here.
-	cfg.EngineArgs["speculative_config"] = map[string]any{"method": "mtp"}
-	xlog.Info("[vllm-spec] MTP head detected; enabling mtp speculative decoding",
-		"name", cfg.Name, "mtp_num_hidden_layers", layers)
-}

core/config/vllm_spec_test.go

@@ -1,117 +0,0 @@
-package config_test
-
-import (
-	. "github.com/mudler/LocalAI/core/config"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("vllm-cpp speculative-decoding auto-defaults", func() {
-	Context("HasSafetensorsMTPHead", func() {
-		It("detects a top-level mtp_num_hidden_layers", func() {
-			n, ok := HasSafetensorsMTPHead([]byte(`{
-				"model_type": "qwen3_5_moe",
-				"mtp_num_hidden_layers": 1
-			}`))
-			Expect(ok).To(BeTrue())
-			Expect(n).To(Equal(uint32(1)))
-		})
-
-		It("detects the head nested under text_config", func() {
-			// Multimodal checkpoints nest the language-model config, which is
-			// where the MTP depth lives (mirrors the engine's own resolution
-			// off config.raw text_config).
-			n, ok := HasSafetensorsMTPHead([]byte(`{
-				"model_type": "qwen3_5_moe",
-				"text_config": {"mtp_num_hidden_layers": 2}
-			}`))
-			Expect(ok).To(BeTrue())
-			Expect(n).To(Equal(uint32(2)))
-		})
-
-		It("reports no head when the key is absent", func() {
-			n, ok := HasSafetensorsMTPHead([]byte(`{"model_type": "llama"}`))
-			Expect(ok).To(BeFalse())
-			Expect(n).To(BeZero())
-		})
-
-		It("reports no head for a zero depth", func() {
-			_, ok := HasSafetensorsMTPHead([]byte(`{"mtp_num_hidden_layers": 0}`))
-			Expect(ok).To(BeFalse())
-		})
-
-		It("ignores a DFlash draft checkpoint", func() {
-			// A DFlash draft is a SEPARATE checkpoint that cannot serve alone:
-			// it needs a target to verify against. Same exclusion the GGUF path
-			// makes for gemma4-assistant drafts.
-			_, ok := HasSafetensorsMTPHead([]byte(`{
-				"model_type": "qwen3_dflash",
-				"mtp_num_hidden_layers": 1,
-				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0, 1]}
-			}`))
-			Expect(ok).To(BeFalse())
-		})
-
-		It("reports no head on unparseable JSON", func() {
-			_, ok := HasSafetensorsMTPHead([]byte(`{not json`))
-			Expect(ok).To(BeFalse())
-		})
-
-		It("reports no head on empty input", func() {
-			_, ok := HasSafetensorsMTPHead(nil)
-			Expect(ok).To(BeFalse())
-		})
-	})
-
-	Context("IsDFlashDraftConfig", func() {
-		It("recognises a draft by its dflash_config block", func() {
-			Expect(IsDFlashDraftConfig([]byte(`{
-				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0]}
-			}`))).To(BeTrue())
-		})
-
-		It("does not flag an ordinary checkpoint", func() {
-			Expect(IsDFlashDraftConfig([]byte(`{"model_type": "qwen3_5_moe"}`))).To(BeFalse())
-		})
-	})
-
-	Context("ApplyVLLMSpeculativeDefaults", func() {
-		It("writes the mtp method into engine_args", func() {
-			cfg := &ModelConfig{Name: "qwen"}
-			ApplyVLLMSpeculativeDefaults(cfg, 1)
-			Expect(cfg.EngineArgs).To(HaveKey("speculative_config"))
-			spec, ok := cfg.EngineArgs["speculative_config"].(map[string]any)
-			Expect(ok).To(BeTrue())
-			Expect(spec["method"]).To(Equal("mtp"))
-		})
-
-		It("leaves an existing speculative_config alone", func() {
-			cfg := &ModelConfig{
-				Name: "qwen",
-				LLMConfig: LLMConfig{
-					EngineArgs: map[string]any{
-						"speculative_config": map[string]any{"method": "ngram", "num_speculative_tokens": 4},
-					},
-				},
-			}
-			ApplyVLLMSpeculativeDefaults(cfg, 1)
-			spec := cfg.EngineArgs["speculative_config"].(map[string]any)
-			Expect(spec["method"]).To(Equal("ngram"))
-		})
-
-		It("preserves unrelated engine_args keys", func() {
-			cfg := &ModelConfig{
-				Name:      "qwen",
-				LLMConfig: LLMConfig{EngineArgs: map[string]any{"max_num_seqs": 32}},
-			}
-			ApplyVLLMSpeculativeDefaults(cfg, 1)
-			Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_seqs", 32))
-			Expect(cfg.EngineArgs).To(HaveKey("speculative_config"))
-		})
-
-		It("tolerates a nil config", func() {
-			Expect(func() { ApplyVLLMSpeculativeDefaults(nil, 1) }).ToNot(Panic())
-		})
-	})
-})

core/gallery/importers/llama-cpp.go

@@ -298,15 +298,7 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
 	// imported configs already carry spec_type:draft-mtp before the model is
 	// ever loaded - users see it in the YAML preview rather than discovering
 	// it after the first start.
-	//
-	// vllm-cpp is excluded on both counts: `spec_type:*` are llama.cpp option
-	// keys it does not read, and vllm.cpp rejects an MTP config over a GGUF
-	// source outright (the `mtp.*` draft tensors exist only in the safetensors
-	// checkpoint). Its MTP auto-config runs in the vllm importer instead, over
-	// the safetensors config.json.
-	if backend != "vllm-cpp" {
-		maybeApplyMTPDefaults(&modelConfig, details, &cfg)
-	}
+	maybeApplyMTPDefaults(&modelConfig, details, &cfg)
 
 	data, err := yaml.Marshal(modelConfig)
 	if err != nil {

core/gallery/importers/vllm.go

@@ -1,21 +1,13 @@
 package importers
 
 import (
-	"context"
 	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/gallery"
 	"github.com/mudler/LocalAI/core/schema"
-	"github.com/mudler/LocalAI/pkg/downloader"
-	"github.com/mudler/LocalAI/pkg/httpclient"
-	"github.com/mudler/xlog"
 	"go.yaml.in/yaml/v2"
 )
 
@@ -115,12 +107,6 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
 		// vllm python backend, so use_tokenizer_template carries over), but
 		// tool/reasoning parsing is the engine's own autoparser pipeline -
 		// the vllm-python tool_parser/reasoning_parser options don't apply.
-		//
-		// Auto-detect a Multi-Token Prediction head, the safetensors analogue
-		// of the llama-cpp importer's GGUF hook, so a freshly imported
-		// Qwen3.5 / Qwen3.6 config already carries speculative decoding in its
-		// engine_args instead of leaving the throughput on the table.
-		maybeApplyVLLMSpeculativeDefaults(&modelConfig, details)
 	} else {
 		// Auto-detect tool_parser and reasoning_parser for known model families.
 		// Surfacing them in the generated YAML lets users see and edit the choices.
@@ -146,89 +132,3 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
 		ConfigFile:  string(data),
 	}, nil
 }
-
-// maxSpecConfigProbeBytes caps the config.json body we read. Real ones are a
-// few KB; the cap keeps a hostile or mislabelled URL from streaming into the
-// importer.
-const maxSpecConfigProbeBytes = 1 << 20 // 1 MiB
-
-// specConfigProbeTimeout bounds the config.json fetch. Detection is an
-// optimisation, so it must never hold an import open for long.
-const specConfigProbeTimeout = 30 * time.Second
-
-// specConfigFetcher is the seam the config.json probe goes through, so tests can
-// drive the whole import path without a network round trip.
-var specConfigFetcher = fetchProbeBody
-
-// maybeApplyVLLMSpeculativeDefaults fetches the repository's config.json and,
-// when it declares a Multi-Token Prediction head, enables MTP speculative
-// decoding in the emitted engine_args. This is the safetensors counterpart of
-// the llama-cpp importer's GGUF header probe.
-//
-// Every failure is non-fatal and logged at debug: a network blip, a private
-// repo, or a config.json this doesn't understand must leave the import working
-// exactly as it did before, just without the speculative default.
-func maybeApplyVLLMSpeculativeDefaults(modelConfig *config.ModelConfig, details Details) {
-	probeURL := vllmSpecProbeURL(details)
-	if probeURL == "" {
-		return
-	}
-
-	body, err := specConfigFetcher(probeURL)
-	if err != nil {
-		xlog.Debug("[vllm-spec-importer] could not read config.json for MTP detection", "uri", probeURL, "error", err)
-		return
-	}
-
-	applySpecFromConfigJSON(modelConfig, body, details.URI)
-}
-
-// applySpecFromConfigJSON is the decision half of the probe, split out so it can
-// be exercised without a network round trip.
-func applySpecFromConfigJSON(modelConfig *config.ModelConfig, body []byte, uri string) {
-	if config.IsDFlashDraftConfig(body) {
-		// A DFlash draft cannot serve on its own - it only proposes tokens for
-		// a target model to verify. Say so rather than emitting a config that
-		// would fail at load.
-		xlog.Warn("[vllm-spec-importer] this repository is a DFlash DRAFT checkpoint, not a servable model; "+
-			"import the TARGET model and point engine_args.speculative_config at this repo "+
-			`({"method":"dflash","model":"<this repo>"})`, "uri", uri)
-		return
-	}
-
-	n, ok := config.HasSafetensorsMTPHead(body)
-	if !ok {
-		return
-	}
-	config.ApplyVLLMSpeculativeDefaults(modelConfig, n)
-}
-
-// vllmSpecProbeURL returns the HTTP(S) URL of the repository's config.json, or
-// "" when the import isn't backed by a HuggingFace repo we can fetch from (a
-// local directory import, an OCI artifact, ...).
-func vllmSpecProbeURL(details Details) string {
-	if details.HuggingFace == nil || details.HuggingFace.ModelID == "" {
-		return ""
-	}
-	return resolveHTTPProbe(downloader.HuggingFacePrefix + details.HuggingFace.ModelID + "/config.json")
-}
-
-// fetchProbeBody GETs a small remote JSON document under a short timeout.
-func fetchProbeBody(url string) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), specConfigProbeTimeout)
-	defer cancel()
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := httpclient.NewWithTimeout(specConfigProbeTimeout).Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer func() { _ = resp.Body.Close() }()
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("unexpected status %d", resp.StatusCode)
-	}
-	return io.ReadAll(io.LimitReader(resp.Body, maxSpecConfigProbeBytes))
-}

core/gallery/importers/vllm_spec_internal_test.go

@@ -1,118 +0,0 @@
-package importers
-
-import (
-	"encoding/json"
-	"errors"
-
-	"github.com/mudler/LocalAI/core/config"
-	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("vllm-cpp speculative auto-config (importer)", func() {
-	Context("applySpecFromConfigJSON", func() {
-		It("enables mtp when the checkpoint declares an MTP head", func() {
-			cfg := &config.ModelConfig{Name: "qwen3.5"}
-			applySpecFromConfigJSON(cfg, []byte(`{
-				"model_type": "qwen3_5_moe",
-				"mtp_num_hidden_layers": 1
-			}`), "huggingface://Qwen/Qwen3.5-A3B")
-			Expect(cfg.EngineArgs).To(HaveKeyWithValue("speculative_config",
-				map[string]any{"method": "mtp"}))
-		})
-
-		It("leaves a plain checkpoint untouched", func() {
-			cfg := &config.ModelConfig{Name: "llama"}
-			applySpecFromConfigJSON(cfg, []byte(`{"model_type": "llama"}`), "huggingface://meta/llama")
-			Expect(cfg.EngineArgs).To(BeEmpty())
-		})
-
-		It("refuses to configure a DFlash draft as a servable model", func() {
-			// The draft only proposes tokens; configuring it standalone would
-			// produce a model that cannot load.
-			cfg := &config.ModelConfig{Name: "dflash-draft"}
-			applySpecFromConfigJSON(cfg, []byte(`{
-				"model_type": "qwen3_dflash",
-				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0, 1]}
-			}`), "huggingface://z-lab/Qwen3.6-27B-DFlash")
-			Expect(cfg.EngineArgs).To(BeEmpty())
-		})
-
-		It("survives a config.json it cannot parse", func() {
-			cfg := &config.ModelConfig{Name: "weird"}
-			Expect(func() {
-				applySpecFromConfigJSON(cfg, []byte(`<html>404</html>`), "huggingface://a/b")
-			}).ToNot(Panic())
-			Expect(cfg.EngineArgs).To(BeEmpty())
-		})
-	})
-
-	Context("Import over a repository with an MTP head", func() {
-		var restore func()
-
-		BeforeEach(func() {
-			original := specConfigFetcher
-			restore = func() { specConfigFetcher = original }
-		})
-		AfterEach(func() { restore() })
-
-		importWith := func(backend, configJSON string) string {
-			specConfigFetcher = func(string) ([]byte, error) {
-				return []byte(configJSON), nil
-			}
-			importer := &VLLMImporter{}
-			out, err := importer.Import(Details{
-				URI:         "huggingface://Qwen/Qwen3.5-A3B",
-				Preferences: json.RawMessage(`{"backend": "` + backend + `"}`),
-				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
-			})
-			Expect(err).ToNot(HaveOccurred())
-			return out.ConfigFile
-		}
-
-		It("emits engine_args.speculative_config for vllm-cpp", func() {
-			yaml := importWith("vllm-cpp", `{"model_type":"qwen3_5_moe","mtp_num_hidden_layers":1}`)
-			Expect(yaml).To(ContainSubstring("engine_args:"))
-			Expect(yaml).To(ContainSubstring("speculative_config:"))
-			Expect(yaml).To(ContainSubstring("method: mtp"))
-		})
-
-		It("emits nothing speculative for the python vllm backend", func() {
-			// The python backend has its own speculative surface and its own
-			// version-dependent MTP support; this hook is vllm-cpp only.
-			yaml := importWith("vllm", `{"model_type":"qwen3_5_moe","mtp_num_hidden_layers":1}`)
-			Expect(yaml).NotTo(ContainSubstring("speculative_config"))
-		})
-
-		It("emits nothing speculative when the probe fails", func() {
-			specConfigFetcher = func(string) ([]byte, error) {
-				return nil, errors.New("network down")
-			}
-			importer := &VLLMImporter{}
-			out, err := importer.Import(Details{
-				URI:         "huggingface://Qwen/Qwen3.5-A3B",
-				Preferences: json.RawMessage(`{"backend": "vllm-cpp"}`),
-				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
-			})
-			Expect(err).ToNot(HaveOccurred())
-			Expect(out.ConfigFile).NotTo(ContainSubstring("speculative_config"))
-		})
-	})
-
-	Context("vllmSpecProbeURL", func() {
-		It("resolves the repository's config.json to an HTTPS URL", func() {
-			url := vllmSpecProbeURL(Details{
-				URI:         "huggingface://Qwen/Qwen3.5-A3B",
-				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
-			})
-			Expect(url).To(ContainSubstring("Qwen/Qwen3.5-A3B"))
-			Expect(url).To(HaveSuffix("config.json"))
-			Expect(url).To(HavePrefix("https://"))
-		})
-
-		It("skips the probe when there is no HuggingFace repo behind the import", func() {
-			Expect(vllmSpecProbeURL(Details{URI: "/models/local-dir"})).To(BeEmpty())
-		})
-	})
-})

core/http/react-ui/package-lock.json

@@ -24,7 +24,7 @@
         "@modelcontextprotocol/sdk": "^1.30.0",
         "dompurify": "^3.4.12",
         "highlight.js": "^11.11.1",
-        "hono": "4.12.25",
+        "hono": "4.12.34",
         "i18next": "^26.0.8",
         "i18next-browser-languagedetector": "^8.2.1",
         "i18next-http-backend": "^3.0.6",
@@ -636,12 +636,12 @@
       }
     },
     "node_modules/@hono/node-server": {
-      "version": "1.19.14",
-      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz",
-      "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.0.tgz",
+      "integrity": "sha512-XovyyCCnBzW+zKu+z/zq8hwNs4KOR5rEMAOxo2f40Q5xoOI37IMm6MIg2COOUtUApo0i6850MTBKH2u4QLGIqg==",
       "license": "MIT",
       "engines": {
-        "node": ">=18.14.1"
+        "node": ">=20"
       },
       "peerDependencies": {
         "hono": "^4"
@@ -2879,9 +2879,9 @@
       "dev": true
     },
     "node_modules/fast-uri": {
-      "version": "3.1.4",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz",
-      "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==",
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
+      "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
       "funding": [
         {
           "type": "github",
@@ -3435,9 +3435,9 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.12.25",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.25.tgz",
-      "integrity": "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==",
+      "version": "4.12.34",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.34.tgz",
+      "integrity": "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA==",
       "license": "MIT",
       "engines": {
         "node": ">=16.9.0"
@@ -4196,9 +4196,9 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
     "node_modules/ip-address": {
-      "version": "10.2.0",
-      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
-      "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==",
+      "version": "10.4.0",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz",
+      "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==",
       "license": "MIT",
       "engines": {
         "node": ">= 12"
@@ -7138,9 +7138,9 @@
       }
     },
     "node_modules/undici": {
-      "version": "7.28.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz",
-      "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==",
+      "version": "7.29.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
+      "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==",
       "dev": true,
       "license": "MIT",
       "engines": {

core/http/react-ui/package.json

@@ -19,7 +19,7 @@
     "coverage:report": "nyc report"
   },
   "overrides": {
-    "hono": "4.12.25"
+    "hono": "4.12.34"
   },
   "dependencies": {
     "@codemirror/autocomplete": "^6.18.6",
@@ -38,7 +38,7 @@
     "@modelcontextprotocol/sdk": "^1.30.0",
     "dompurify": "^3.4.12",
     "highlight.js": "^11.11.1",
-    "hono": "4.12.25",
+    "hono": "4.12.34",
     "i18next": "^26.0.8",
     "i18next-browser-languagedetector": "^8.2.1",
     "i18next-http-backend": "^3.0.6",

core/services/advisorylock/leader_loop_test.go

@@ -54,62 +54,57 @@ var _ = Describe("RunLeaderLoop", func() {
 				close(done)
 			}()
 
-			// Let it run a bit then cancel
-			time.Sleep(150 * time.Millisecond)
+			Eventually(func() int32 {
+				return atomic.LoadInt32(&callCount)
+			}, 500*time.Millisecond, 10*time.Millisecond).Should(BeNumerically(">=", 1))
 			cancel()
 
-			// RunLeaderLoop should return
 			Eventually(done, 500*time.Millisecond).Should(BeClosed())
-
-			// Record count after cancellation
-			countAfterCancel := atomic.LoadInt32(&callCount)
-			time.Sleep(150 * time.Millisecond)
-			countLater := atomic.LoadInt32(&callCount)
-
-			Expect(countLater).To(Equal(countAfterCancel),
-				"function should stop being called after context cancellation")
 		})
 
 		It("only one leader executes at a time (two concurrent loops)", func() {
 			db := testutil.SetupTestDB()
 			const lockKey int64 = 5002
 
-			var (
-				mu         sync.Mutex
-				maxRunning int32
-				running    int32
-			)
+			var running int32
+			entered := make(chan struct{}, 2)
+			release := make(chan struct{})
+			var releaseOnce sync.Once
 
 			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
+			done := make(chan struct{}, 2)
+			DeferCleanup(func() {
+				cancel()
+				releaseOnce.Do(func() { close(release) })
+			})
 
 			fn := func() {
-				cur := atomic.AddInt32(&running, 1)
-				mu.Lock()
-				if cur > maxRunning {
-					maxRunning = cur
+				atomic.AddInt32(&running, 1)
+				select {
+				case entered <- struct{}{}:
+				default:
 				}
-				mu.Unlock()
-
-				time.Sleep(30 * time.Millisecond)
-
+				<-release
 				atomic.AddInt32(&running, -1)
 			}
 
-			// Start two competing leader loops with the same lock key
-			go RunLeaderLoop(ctx, db, lockKey, 50*time.Millisecond, fn)
-			go RunLeaderLoop(ctx, db, lockKey, 50*time.Millisecond, fn)
+			for range 2 {
+				go func() {
+					RunLeaderLoop(ctx, db, lockKey, 1*time.Millisecond, fn)
+					done <- struct{}{}
+				}()
+			}
+
+			Eventually(entered, 500*time.Millisecond).Should(Receive())
+			Consistently(func() int32 {
+				return atomic.LoadInt32(&running)
+			}, 50*time.Millisecond, 5*time.Millisecond).Should(Equal(int32(1)),
+				"expected only the lock holder to run while both loops tick")
 
-			// Let them run for a while
-			time.Sleep(400 * time.Millisecond)
 			cancel()
-
-			mu.Lock()
-			observed := maxRunning
-			mu.Unlock()
-
-			Expect(observed).To(BeNumerically("<=", 1),
-				"expected at most 1 goroutine running the leader function at a time")
+			releaseOnce.Do(func() { close(release) })
+			Eventually(done, 500*time.Millisecond).Should(Receive())
+			Eventually(done, 500*time.Millisecond).Should(Receive())
 		})
 	})
 })

docs/content/features/text-generation.md

@@ -918,200 +918,6 @@ options:
 The full list of registered parsers lives in `sglang.srt.function_call`
 and `sglang.srt.parser.reasoning_parser`.
 
-### vllm.cpp
-
-[vllm.cpp](https://github.com/mudler/vllm.cpp) is the LocalAI team's C++ port of
-vLLM: the same continuous-batching scheduler, paged KV cache and prefix caching,
-with no Python at inference time. It consumes either a HuggingFace safetensors
-model directory or a `.gguf` file, and applies the model's chat template,
-tool-call parsing and reasoning split engine-side.
-
-#### Setup
-
-```yaml
-name: vllm-cpp
-backend: vllm-cpp
-parameters:
-  model: "Qwen/Qwen3-4B"
-context_size: 8192
-template:
-  use_tokenizer_template: true
-```
-
-#### Configuring the engine with `engine_args`
-
-The same `engine_args:` map the vLLM and SGLang backends accept is honoured
-here, with keys spelled exactly as vLLM's own CLI flags - so a `speculative_config`
-or `kv_transfer_config` block written for vLLM works verbatim. Unknown keys are
-ignored rather than fatal; the engine validates the documents it is handed and
-reports a precise error at load.
-
-```yaml
-name: qwen35-a3b
-backend: vllm-cpp
-parameters:
-  model: "Qwen/Qwen3.5-A3B"
-context_size: 16384
-template:
-  use_tokenizer_template: true
-engine_args:
-  # KV cache sizing: num_blocks * block_size tokens of cache.
-  block_size: 32
-  num_blocks: 1024
-  # Concurrency and the per-step chunked-prefill token budget.
-  max_num_seqs: 32
-  max_num_batched_tokens: 8192
-  # Automatic prefix caching. Omit to keep the model's own default
-  # (on for dense models, off for hybrid / attention-free ones).
-  enable_prefix_caching: true
-  # Scheduler admission order: fcfs (default), priority, or lpm
-  # (cache-aware longest-prefix-match; needs prefix caching to have any effect).
-  scheduling_policy: lpm
-```
-
-| Key | Meaning | Default |
-|-----|---------|---------|
-| `block_size` | KV-cache block size, in tokens per block | 32 |
-| `num_blocks` | KV-cache blocks to allocate | 256 |
-| `max_model_len` | Max sequence length; also settable as `context_size` / `max_model_len` | model config |
-| `max_num_seqs` | Max concurrent sequences the scheduler admits | 8 |
-| `max_num_batched_tokens` | Per-step chunked-prefill token budget | per-arch (2048 dense, 4096/8192 MoE) |
-| `enable_prefix_caching` | Automatic prefix caching; `enable_radix_attention` is an accepted alias | model default |
-| `enable_jump_forward` | Jump-forward decoding, which emits grammar-forced tokens without a model step. Only affects constrained requests (`grammar`, JSON schema) | off |
-| `scheduling_policy` | `fcfs`, `priority`, or `lpm` | `fcfs` |
-| `tool_parser` / `reasoning_parser` | Force a parser instead of chat-template auto-detection | auto |
-| `tokenizer_config` | Override the `tokenizer_config.json` the chat template is read from | `<model_dir>/tokenizer_config.json` |
-| `speculative_config` | Speculative decoding (see below) | disabled |
-| `kv_transfer_config` | External KV connector / LMCache (see below) | none |
-
-Raising `max_num_batched_tokens` lets more prefill land in a single step, at the
-cost of decode latency for requests queued behind it. The default deliberately
-does not scale with `max_num_seqs`, which is what keeps a large concurrent
-prefill from blowing up the per-step activation on the hybrid architectures.
-
-`enable_prefix_caching` and `enable_jump_forward` are tri-state at the engine
-boundary: omitting the key defers to a default (the model's own capability for
-prefix caching, an environment variable for jump forward), while an explicit
-`false` forces the feature off. Those are genuinely different - prefix caching
-defaults *on* for dense models - so write the key only when you mean to override.
-
-#### Speculative decoding
-
-`speculative_config:` takes the same JSON object as vLLM's
-`--speculative-config`. Three methods are supported.
-
-> **Architecture limit.** At the current engine pin, `mtp` and `dflash` are
-> **Qwen3.5 / Qwen3.6 only**. The engine builds a widened speculative KV cache
-> directly for those families rather than through the model registry, so a
-> speculative config on any other architecture (Llama, GLM, Gemma, Mistral, ...)
-> will not work regardless of checkpoint format. `ngram` needs no draft weights
-> and is not subject to this limit.
-
-> **Format support.** `mtp` and `dflash` now work from a `.gguf` target as well
-> as safetensors. An MTP head is read from the GGUF's `nextn.*` tensors when the
-> file declares `<arch>.nextn_predict_layers`; a GGUF exported WITHOUT the head
-> (converted with `--no-mtp`, or predating llama.cpp's Qwen3.5 MTP support) is
-> refused at load naming that as the reason. A DFlash draft may itself be a
-> `dflash`-arch GGUF, and the target may be a GGUF too. `ngram` needs no draft
-> weights and works on any format.
-
-**MTP** (Multi-Token Prediction) uses a draft head shipped inside the target
-checkpoint's own `mtp.*` tensors, so there is no second model to download. It
-requires a **safetensors** checkpoint - the `mtp.*` tensors do not survive GGUF
-conversion, and an MTP config over a `.gguf` model is rejected at load.
-
-```yaml
-engine_args:
-  speculative_config:
-    method: mtp
-    # Optional; defaults to the checkpoint's own head depth, which is
-    # usually the right value. Must be a multiple of that depth.
-    num_speculative_tokens: 1
-```
-
-**DFlash** uses a separate block-diffusion drafter that proposes a whole block
-of tokens in one non-autoregressive forward pass. Unlike MTP, the draft is its
-own checkpoint, so `model:` is **required**:
-
-```yaml
-engine_args:
-  speculative_config:
-    method: dflash
-    model: z-lab/Qwen3.6-27B-DFlash
-    num_speculative_tokens: 4
-```
-
-The draft shares the *target's* `embed_tokens` and `lm_head`, so both must come
-from the same model family and the target must be safetensors.
-
-**The engine does not download the draft.** `model:` is resolved, in order,
-as a path as given, then as the last path segment under LocalAI's models
-directory (`z-lab/Qwen3.6-27B-DFlash` → `<models>/Qwen3.6-27B-DFlash`, which is
-what LocalAI's own downloader produces), then as the whole reference under the
-models directory. Install the draft into LocalAI first, or give an absolute path
-to a directory containing `config.json`. If none of those resolve, the load
-fails immediately naming every location that was tried, rather than reporting a
-missing checkpoint from inside the engine.
-
-**N-gram** needs no draft model at all - it proposes from the prompt's own
-suffix history. `num_speculative_tokens` is required:
-
-```yaml
-engine_args:
-  speculative_config:
-    method: ngram
-    num_speculative_tokens: 4
-    prompt_lookup_min: 5
-    prompt_lookup_max: 5
-```
-
-> **Auto-configuration on import.** When you import a safetensors repository
-> with `backend: vllm-cpp`, LocalAI reads the checkpoint's `config.json` and, if
-> it declares an MTP head (`mtp_num_hidden_layers`), writes
-> `speculative_config: {method: mtp}` into the generated `engine_args` for you.
-> An explicit `speculative_config` in your own config is never overwritten.
-> Importing a DFlash *draft* repository is refused with a warning: a drafter
-> cannot serve on its own, so import the target model and point
-> `speculative_config.model` at the draft.
-
-#### External KV cache with LMCache
-
-`kv_transfer_config:` takes vLLM's `--kv-transfer-config` JSON and selects an
-external KV-cache connector. The `lm://` LMCache client lets prefill KV be
-stored to and reloaded from a shared `lmcache.v1.server`, so a prefix computed
-by one replica does not have to be recomputed by the next:
-
-```yaml
-engine_args:
-  kv_transfer_config:
-    kv_connector: LMCacheConnector
-    kv_role: kv_both          # required whenever kv_connector is set
-    kv_connector_extra_config:
-      host: 127.0.0.1
-      port: 65432
-```
-
-`kv_role` is one of `kv_producer` (store only), `kv_consumer` (load only), or
-`kv_both`. An unregistered connector name, a missing role, or a malformed
-document fails the load with an explicit error rather than silently running
-without the cache.
-
-#### Legacy `options:` list
-
-Earlier versions configured this backend through the flat `options:` list, and
-those configs keep working. Every key in the table above is still read from
-there in `key:value` form, and `engine_args` wins on any key set in both:
-
-```yaml
-options:
-  - max_num_seqs:32
-  - enable_prefix_caching:true
-```
-
-New configs should prefer `engine_args:`, which is the only place the nested
-`speculative_config` / `kv_transfer_config` documents can be written naturally
-rather than as a single-line JSON string.
-
 ### Transformers
 
 [Transformers](https://huggingface.co/docs/transformers/index) is a State-of-the-art Machine Learning library for PyTorch, TensorFlow, and JAX.

gallery/index.yaml

@@ -785,35 +785,18 @@
 - name: "qwen3.6-35b-a3b-uncensored-genesis-hermes-v6"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
+    - https://huggingface.co/HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive
     - https://huggingface.co/LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V6-GGUF
   description: |
-    # Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive
+    Qwen3.6-35B-A3B Uncensored Genesis Hermes V6 is LuffyTheFox's multimodal,
+    agentic derivative of HauhauCS's uncensored Qwen3.6-35B-A3B model. It
+    combines Genesis tensor calibration with Hermes function-calling data while
+    retaining the 35B mixture-of-experts architecture, roughly 3B active
+    parameters per token, and the native 262K-token context window.
 
-    > **Join the Discord** for updates, roadmaps, projects, or just to chat.
-
-    Qwen3.6-35B-A3B uncensored by HauhauCS. **0/465 Refusals.**
-
-    > **HuggingFace's "Hardware Compatibility" widget doesn't recognize K_P quants** — it may show fewer files than actually exist. Click **"View +X variants"** or go to **Files and versions** to see all available downloads.
-
-    ## About
-
-    No changes to datasets or capabilities. Fully functional, 100% of what the original authors intended - just without the refusals.
-
-    These are meant to be the best lossless uncensored models out there.
-
-    ## Aggressive Variant
-
-    Stronger uncensoring — model is fully unlocked and won't refuse prompts. May occasionally append short disclaimers (baked into base model training, not refusals) but full content is always generated.
-
-    For a more conservative uncensor that keeps some safety guardrails, check the Balanced variant when it's available.
-
-    ## Downloads
-
-    All quants generated with importance matrix (imatrix) for optimal quality preservation on abliterated weights.
-
-    ## What are K_P quants?
-
-    ...
+    This entry installs the Q8_0 GGUF together with its F16 multimodal projector
+    for llama.cpp. The model card recommends Jinja chat templates and at least a
+    128K context for its thinking behavior. License: Apache-2.0.
   license: "apache-2.0"
   tags:
     - llm

scripts/build/llama-cpp-build-target_test.sh

@@ -29,4 +29,11 @@ assert_target arm64 "" llama-cpp-cpu-all
 assert_target amd64 sycl_f16 llama-cpp-fallback
 assert_target amd64 sycl_f32 llama-cpp-fallback
 
+# ROCm exhausts the same 6h budget through volume rather than a stall: hipcc
+# compiles ggml's HIP kernels once per AMDGPU target, eleven of them, and the
+# CPU variant matrix goes on top. 2h27m before it was added, killed at exactly
+# 6h00m on every run since.
+assert_target amd64 hipblas llama-cpp-fallback
+assert_target arm64 hipblas llama-cpp-fallback
+
 echo "PASS: llama.cpp build target preserves CPU variants where supported"

scripts/lib/backend-filter.mjs

@@ -436,6 +436,27 @@ export const SHARED_BUILD_INPUTS = [
     linux: always,
     darwin: always,
   },
+  {
+    // Same posture as the scripts/build/ catch-all above, and for the same
+    // reason. .docker/ holds the per-backend compile and build-target scripts
+    // (llama-cpp, turboquant, bonsai, ik-llama-cpp) plus inputs every
+    // Dockerfile consumes (apt-mirror.sh, install-base-deps.sh). Nothing
+    // matched any of them before, which is how #11346 shipped without a single
+    // backend job: it changed how ROCm llama.cpp compiles, touching only
+    // .docker/llama-cpp-build-target.sh and a `*_test.sh` that the rule above
+    // deliberately carves out, so the filter selected zero entries and every
+    // backend job reported "skipping".
+    //
+    // A rule cannot see which file matched it, only the matrix entry, so
+    // narrowing `.docker/<name>-compile.sh` to the backend named by its prefix
+    // would mean threading the filename through matchedSharedRules. Until
+    // someone wants that, take the full matrix: these files are edited a
+    // handful of times a release, and a shared build input silently shipping
+    // to nothing is the failure this list exists to prevent.
+    matches: file => file.startsWith(".docker/"),
+    linux: always,
+    darwin: always,
+  },
 ];
 
 // The matrix stores dockerfiles as "./backend/Dockerfile.python"; changed-file

scripts/lib/backend-filter_test.mjs

@@ -204,6 +204,29 @@ test("an unclassified scripts/build/ file conservatively rebuilds everything", (
   assert.equal(filteredDarwin.length, includesDarwin.length);
 });
 
+// #11346 changed how ROCm llama.cpp compiles and built nothing: it touched only
+// .docker/llama-cpp-build-target.sh and a *_test.sh, no rule matched either, so
+// every backend job reported "skipping".
+test("a .docker/ compile script rebuilds the backends it compiles", () => {
+  const { filtered } = run([
+    ".docker/llama-cpp-build-target.sh",
+    "scripts/build/llama-cpp-build-target_test.sh",
+  ]);
+
+  assert.ok(filtered.length > 0, ".docker/ change selected no entries");
+  assert.ok(
+    filtered.some(e => e.backend === "llama-cpp"),
+    "llama-cpp was not selected by a change to its own compile script",
+  );
+});
+
+test("a shared .docker/ input rebuilds everything", () => {
+  const { filtered, filteredDarwin } = run([".docker/apt-mirror.sh"]);
+
+  assert.equal(filtered.length, includes.length);
+  assert.equal(filteredDarwin.length, includesDarwin.length);
+});
+
 test("tests for the packaging scripts do not rebuild anything", () => {
   const { filtered, filteredDarwin } = run([
     "scripts/build/package-gpu-libs_test.sh",