fix(vllm-cpp): pass Darwin warning flag to Objective-C

The Metal source that triggers Apple Clang's constant-folding diagnostic is compiled as Objective-C. Pass the targeted suppression through CMAKE_OBJC_FLAGS as well as the C++ language flags so vllm.cpp's -Werror no longer promotes it.

Assisted-by: Codex:gpt-5

.docker/llama-cpp-build-target.sh

@@ -8,15 +8,8 @@ build_type=${2-}
 # ggml-cpu/arch/x86/repack.cpp at -march=sapphirerapids: the job sits on that one
 # translation unit until GitHub kills it at 6h. gcc builds the same file in
 # seconds, so only the SYCL images have to give up the CPU variant matrix.
-#
-# ROCm runs out of the same 6h budget for a different reason: volume, not a
-# stall. hipcc compiles ggml's HIP kernels once per entry in AMDGPU_TARGETS,
-# which is eleven architectures (gfx908 through gfx1201), and the CPU variant
-# matrix lands on top of that. The job built in 2h27m before it was added and
-# has been killed at exactly 6h00m on every run since, so no ROCm llama-cpp
-# image has been published since 2026-08-01.
 case "$build_type" in
-  sycl*|hipblas*)
+  sycl*)
     echo llama-cpp-fallback
     exit 0
     ;;

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=238ab6a9e321c17de8e120559f57efeedaeb1345
+AUDIO_CPP_VERSION?=4e3aea2fd99aeaa5924e71c51eb2793846045332
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ds4/CMakeLists.txt

@@ -69,15 +69,7 @@ target_include_directories(hw_grpc_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
 
 set(DS4_OBJS "${DS4_DIR}/ds4.o")
 if(DS4_GPU STREQUAL "cuda")
-    list(APPEND DS4_OBJS
-        "${DS4_DIR}/ds4_cuda.o"
-        "${DS4_DIR}/cuda/mmq/ds4_ggml_stubs.o"
-        "${DS4_DIR}/cuda/mmq/ds4_mmq.o"
-        "${DS4_DIR}/cuda/mmq/ds4_mmq_d2r.o"
-        "${DS4_DIR}/cuda/mmq/quantize.o"
-        "${DS4_DIR}/cuda/mmq/mmid.o"
-        "${DS4_DIR}/cuda/mmq/mmvq.o"
-        "${DS4_DIR}/cuda/mmq/ds4_repack.o")
+    list(APPEND DS4_OBJS "${DS4_DIR}/ds4_cuda.o")
 elseif(DS4_GPU STREQUAL "metal")
     list(APPEND DS4_OBJS "${DS4_DIR}/ds4_metal.o")
 elseif(DS4_GPU STREQUAL "cpu")

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=6747e7718dd08f00b680d0c16231f2d59ec3747e
+# Upstream pin lives below as DS4_VERSION?=54b36ed9ba42da31b24f2d1a5feb075c2475dbb1
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=6747e7718dd08f00b680d0c16231f2d59ec3747e
+DS4_VERSION?=54b36ed9ba42da31b24f2d1a5feb075c2475dbb1
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -23,9 +23,7 @@ CMAKE_ARGS ?= -DCMAKE_BUILD_TYPE=Release
 # are shared by every GPU mode, so append them unconditionally below.
 ifeq ($(BUILD_TYPE),cublas)
     CMAKE_ARGS += -DDS4_GPU=cuda
-    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \
-        cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \
-        cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o
+    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else ifeq ($(UNAME_S),Darwin)
     CMAKE_ARGS += -DDS4_GPU=metal
     DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
@@ -57,7 +55,7 @@ ds4:
 # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA).
 ds4/ds4.o: ds4
 ifeq ($(BUILD_TYPE),cublas)
-	+$(MAKE) -C ds4 $(DS4_OBJ_TARGET)
+	+$(MAKE) -C ds4 ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else ifeq ($(UNAME_S),Darwin)
 	+$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=6b55d2c7504f482e7c8ec6cbf22a19f3778c522b
+IK_LLAMA_VERSION?=60389410a1ff01f9d37dcc6261db33b3183bdea2
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=ec730908a418b6032f9e69ded6186d3f042a7747
+CRISPASR_VERSION?=fe3caf8e363b27572dbdd1a9d37083f25e6decda
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=ea7f0c87cfe4c673263b4c201c596c7f1cbe2528
+STABLEDIFFUSION_GGML_VERSION?=db99efdd6d2a43c7937fd55b3359206c680a75b0
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/vllm-cpp/Makefile

@@ -11,30 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=0757cac231ecd571a83c4fd2f50805c9251fc225
-
-# MLX GEMM provider (darwin/metal only; see the metal branch below for why).
-# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun
-# metal`, i.e. a full Xcode the macOS runners do not have, while the wheel ships
-# include/, lib/libmlx.dylib and the compiled mlx.metallib ready to link.
-#
-# DEFAULT ON, but ONLY because VLLM_CPP_VERSION above is pinned at or past
-# vllm.cpp 89c46aeb, which SHAPE-GATES the provider to prefill. The ordering is
-# load-bearing, not incidental:
-#
-#   pin >= 89c46aeb, MLX on   -> 99.1% of MLX-LM  (gated: prefill only)
-#   pin <  89c46aeb, MLX on   -> ~51%             (ungated: it also takes decode)
-#
-# MLX's steel GEMM wins prefill (537 ms TTFT against 602) and loses decode badly,
-# because the provider pays an mx::eval sync plus an output memcpy per call and
-# decode makes ~112 calls per TOKEN. Ungated it does both; gated it does only the
-# good half. So if this pin is ever moved BACKWARDS, this default must go with it.
-VLLM_CPP_MLX?=on
-MLX_VERSION?=0.29.4
-MLX_VENV?=$(abspath ./mlx-venv)
-# Resolved lazily (recursive `=`, not `:=`): the glob only matches once the venv
-# target has run, and the interpreter version in the path varies per runner.
-MLX_ROOT=$(shell echo $(MLX_VENV)/lib/python*/site-packages/mlx)
+VLLM_CPP_VERSION?=a42b8187caff02c570c28e19e4dc2b1d7f55ed14
 
 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
 # server, examples and tests of the engine are never built here.
@@ -72,23 +49,6 @@ else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF
 else ifeq ($(BUILD_TYPE),metal)
 	CMAKE_ARGS+=-DVLLM_CPP_METAL=ON
-	# The optional MLX GEMM provider. vllm.cpp keeps it OFF by default because it
-	# is a ~19 MB libmlx.dylib plus a ~105 MB mlx.metallib, and upstream's
-	# position is that it must earn that cost by measurement. It does, on the
-	# only hardware this build targets: measured on an Apple M4 against the
-	# native MSL GEMM in the SAME binary (arms toggled by
-	# VT_OP_PROVIDER_DISABLE=mlx), Qwen3-1.7B-bf16 p=512 g=128, it is 1.5x to
-	# 2.2x aggregate throughput and 2x to 3x faster TTFT, at equal peak memory
-	# and bit-identical output on every parity shape. See vllm.cpp
-	# docs/BENCHMARKS.md "MLX GEMM provider A/B on Apple M4".
-	#
-	# MLX delegates the dense GEMM ONLY: kPagedAttention stays vllm.cpp's own
-	# kernel, because MLX has no paged-KV primitive at all.
-	#
-	# Set VLLM_CPP_MLX=off for a Metal build without it (smaller image, slower).
-	ifeq ($(VLLM_CPP_MLX),on)
-		MLX_ENABLED=1
-	endif
 else
 	CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF
 endif
@@ -96,6 +56,12 @@ endif
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Darwin)
 	LIB=libvllm.dylib
+	# Apple Clang diagnoses a pair of constant-folded array bounds in the Metal
+	# build as a GNU extension. Disable that diagnostic for both Objective-C and
+	# C++ because vllm.cpp appends target-local -Werror after these global flags.
+	CMAKE_ARGS+=-DCMAKE_CXX_FLAGS=-Wno-gnu-folding-constant
+	CMAKE_ARGS+=-DCMAKE_OBJC_FLAGS=-Wno-gnu-folding-constant
+	CMAKE_ARGS+=-DCMAKE_OBJCXX_FLAGS=-Wno-gnu-folding-constant
 else
 	LIB=libvllm.so
 endif
@@ -108,35 +74,10 @@ sources/vllm.cpp:
 	git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \
 	git checkout FETCH_HEAD
 
-ifeq ($(MLX_ENABLED),1)
-# A stamp FILE, not a phony target: a phony prerequisite is always "newer" than
-# $(LIB) and would re-link libvllm on every invocation. Keyed on the version so
-# a MLX_VERSION bump reinstalls instead of silently reusing the old wheel.
-MLX_STAMP=$(MLX_VENV)/.mlx-$(MLX_VERSION).stamp
-MLX_CMAKE_ARGS=-DVLLM_CPP_MLX=ON -DMLX_ROOT=$(MLX_ROOT)
-
-$(MLX_STAMP):
-	@if [ ! -x "$(MLX_VENV)/bin/pip" ]; then \
-		python3 -m venv "$(MLX_VENV)" || { echo "vllm-cpp: python3 with venv is required to build the MLX provider; pass VLLM_CPP_MLX=off to build Metal without it" >&2; exit 1; }; \
-	fi
-	"$(MLX_VENV)"/bin/pip install --quiet --disable-pip-version-check "mlx==$(MLX_VERSION)"
-	@# Resolved in the SHELL, not by $(MLX_ROOT): make expands a whole recipe
-	@# before running its first line, so the glob would still be unmatched here.
-	@# Every later use (the cmake args, package.sh) expands after this target has
-	@# completed, where $(MLX_ROOT) does resolve.
-	@root=$$(echo "$(MLX_VENV)"/lib/python*/site-packages/mlx); \
-	test -f "$$root/lib/libmlx.dylib" -a -f "$$root/include/mlx/array.h" || \
-		{ echo "vllm-cpp: mlx==$(MLX_VERSION) did not provide lib/libmlx.dylib + include/mlx/array.h under $$root" >&2; exit 1; }
-	touch $@
-else
-MLX_STAMP=
-MLX_CMAKE_ARGS=
-endif
-
-$(LIB): sources/vllm.cpp $(MLX_STAMP)
+$(LIB): sources/vllm.cpp
 	mkdir -p build && \
 	cd build && \
-	cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \
+	cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \
 	cmake --build . --config Release -j$(JOBS) --target vllm_shared
 	cp -fL build/$(LIB) ./$(LIB)
 
@@ -144,12 +85,12 @@ vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB)
 	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./
 
 package: vllm-cpp
-	MLX_ROOT="$(MLX_ROOT)" bash package.sh
+	bash package.sh
 
 build: package
 
 clean: purge
-	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp "$(MLX_VENV)"
+	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp
 
 purge:
 	rm -rf build

backend/go/vllm-cpp/README.md

@@ -41,50 +41,5 @@ options:
 - max_num_seqs:16
 ```
 
-## Apple Silicon: the MLX GEMM provider (ON by default, gated to prefill)
-
-`BUILD_TYPE=metal` builds vllm.cpp's MLX provider for the dense GEMM
-(`VLLM_CPP_MLX=on`, the default here). It is on because upstream now SHAPE-GATES
-it to prefill; it was briefly off in this branch's history, and that was correct
-at the time for an ungated provider.
-
-The gate matters more than the flag. MLX's steel GEMM wins prefill but loses
-decode, because the provider pays an `mx::eval` synchronisation plus an output
-memcpy on every call and decode makes ~112 calls *per token*. Measured on an
-Apple M4, Qwen3-1.7B-bf16 warm at p=512 g=128:
-
-| configuration | prefill TTFT | warm throughput |
-|---|--:|--:|
-| MLX **gated to prefill** (pin >= 89c46aeb) | **524.5 ms** | **24.37 tok/s, 97.6% of MLX-LM** |
-| MLX ungated (older pins) | 537 ms | 12.7 tok/s |
-| MLX off | 602 ms | 23.9 tok/s, 95.9% |
-
-Ratios are against an MLX-LM baseline measured INTERLEAVED with ours over four
-ABBA blocks (its spread 0.34%, ours 0.12%). An earlier revision of this file
-claimed 99.1%; that used a two-run MLX-LM baseline containing an outlier and
-overstated us by about 1.5 points.
-
-**`VLLM_CPP_VERSION` and this flag are coupled.** Moving the pin back before
-`89c46aeb` while leaving `VLLM_CPP_MLX=on` would take the middle row — roughly
-half throughput. If you roll the pin back, roll the default back with it.
-
-One caveat: MLX's GEMM is not bit-identical to the native kernel, so an MLX build
-produces a different greedy sequence than a non-MLX one. That is a property of the
-provider, not of the gate, and it predates this packaging. Full disposition in
-vllm.cpp `docs/BENCHMARKS.md`.
-
-Build knobs:
-
-- `VLLM_CPP_MLX=off` builds Metal without the provider: ~124 MB smaller, and
-  96.4% of MLX-LM instead of 99.1%.
-- `MLX_VERSION` pins the wheel (default `0.29.4`). MLX is consumed as the
-  prebuilt pip wheel because building it from source needs `xcrun metal`, i.e. a
-  full Xcode the macOS runners do not have.
-
-Packaging vendors `libmlx.dylib`, `mlx.metallib` and MLX's MIT license into
-`package/lib/`, and rewrites `libvllm.dylib`'s rpath to `@loader_path/lib`
-(re-signing it, since `install_name_tool` invalidates the signature). The
-metallib must stay beside `libmlx.dylib`: MLX looks for it there.
-
 Testing: `make test` runs the unit specs; export `VLLM_CPP_MODEL=<model>` (and
 optionally `VLLM_CPP_LIBRARY=<libvllm path>`) to enable the e2e specs.

backend/go/vllm-cpp/backend.go

@@ -109,6 +109,16 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 
 	v.opts = parseOptions(opts)
 
+	// A DFlash draft is a second checkpoint the engine opens by path, and the
+	// engine never downloads one. Resolve it against LocalAI's models directory
+	// now so a repo-id spelling works, and so a missing draft fails here with an
+	// actionable message rather than as an HF-cache miss inside the load.
+	resolvedSpec, err := resolveDraftModelPath(v.opts.speculativeConfig, opts.ModelPath)
+	if err != nil {
+		return err
+	}
+	v.opts.speculativeConfig = resolvedSpec
+
 	mp := defaultModelParams()
 	if v.opts.blockSize > 0 {
 		mp.BlockSize = v.opts.blockSize
@@ -116,34 +126,62 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 	if v.opts.numBlocks > 0 {
 		mp.NumBlocks = v.opts.numBlocks
 	}
+	// Sequence-length precedence, narrowest source last: context_size is the
+	// generic LocalAI knob every backend honours, max_model_len is the
+	// vLLM-specific one, and engine_args.max_model_len is the explicit
+	// vllm-cpp override.
 	if opts.ContextSize > 0 {
 		mp.MaxModelLen = opts.ContextSize
 	}
+	if opts.MaxModelLen > 0 {
+		mp.MaxModelLen = opts.MaxModelLen
+	}
+	if v.opts.maxModelLen > 0 {
+		mp.MaxModelLen = v.opts.maxModelLen
+	}
 	if v.opts.maxNumSeqs > 0 {
 		mp.MaxNumSeqs = v.opts.maxNumSeqs
 	}
+	if v.opts.maxNumBatchedTokens > 0 {
+		mp.MaxNumBatchedTokens = v.opts.maxNumBatchedTokens
+	}
+	mp.EnablePrefixCaching = v.opts.enablePrefixCaching
+	mp.EnableJumpForward = v.opts.enableJumpForward
 
+	// Every string below is borrowed by C for the duration of the load call
+	// only (the library copies what it keeps), so the backing slices just have
+	// to outlive vllmEngineLoad - hence the single KeepAlive after it.
 	modelC := cString(model)
 	mp.ModelPath = uintptr(unsafe.Pointer(&modelC[0])) // #nosec G103 -- borrowed by C for the load call only
-	var toolParserC, reasoningParserC []byte
-	if v.opts.toolParser != "" {
-		toolParserC = cString(v.opts.toolParser)
-		mp.ToolParser = uintptr(unsafe.Pointer(&toolParserC[0])) // #nosec G103 -- borrowed by C for the load call only
-	}
-	if v.opts.reasoningParser != "" {
-		reasoningParserC = cString(v.opts.reasoningParser)
-		mp.ReasoningParser = uintptr(unsafe.Pointer(&reasoningParserC[0])) // #nosec G103 -- borrowed by C for the load call only
+	keep := [][]byte{modelC}
+	setStr := func(dst *uintptr, s string) {
+		if s == "" {
+			return
+		}
+		b := cString(s)
+		keep = append(keep, b)
+		*dst = uintptr(unsafe.Pointer(&b[0])) // #nosec G103 -- borrowed by C for the load call only
 	}
+	setStr(&mp.ToolParser, v.opts.toolParser)
+	setStr(&mp.ReasoningParser, v.opts.reasoningParser)
+	setStr(&mp.SpeculativeConfig, v.opts.speculativeConfig)
+	setStr(&mp.KVTransferConfig, v.opts.kvTransferConfig)
+	setStr(&mp.SchedulingPolicy, v.opts.schedulingPolicy)
+	setStr(&mp.TokenizerConfigPath, v.opts.tokenizerConfigPath)
 
 	xlog.Info("[vllm-cpp] Load", "model", model, "engine", vllmVersion(),
 		"blockSize", mp.BlockSize, "numBlocks", mp.NumBlocks,
-		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs)
+		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs,
+		"maxNumBatchedTokens", mp.MaxNumBatchedTokens,
+		"prefixCaching", triStateName(mp.EnablePrefixCaching),
+		"jumpForward", triStateName(mp.EnableJumpForward),
+		"schedulingPolicy", v.opts.schedulingPolicy,
+		"speculativeConfig", v.opts.speculativeConfig,
+		"kvTransferConfig", v.opts.kvTransferConfig)
 
 	var engine uintptr
 	rc := vllmEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params
-	runtime.KeepAlive(modelC)
-	runtime.KeepAlive(toolParserC)
-	runtime.KeepAlive(reasoningParserC)
+	runtime.KeepAlive(keep)
 	if rc != vllmOK {
 		return fmt.Errorf("vllm-cpp: engine load failed: %s", vllmLastError())
 	}

backend/go/vllm-cpp/govllmcpp.go

@@ -1,6 +1,6 @@
 package main
 
-// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v2).
+// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10).
 //
 // The structs below are hand-mirrored PODs of the C declarations, with
 // explicit padding so the Go layout matches the C layout on linux/darwin
@@ -18,23 +18,56 @@ import (
 )
 
 // abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h).
-const abiVersion = 5
+const abiVersion = 10
+
+// The ABI's tri-state toggles (enable_prefix_caching ABI v7,
+// enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
+// "defer" - to the model capability for prefix caching, to the environment for
+// jump forward. Only 2 is an explicit off.
+const (
+	triStateDefer int32 = 0
+	triStateOn    int32 = 1
+	triStateOff   int32 = 2
+)
+
+// triStateName renders a tri-state for the load log line, where "0" would
+// otherwise read as "off" rather than "whatever the default resolves to".
+func triStateName(state int32) string {
+	switch state {
+	case triStateOn:
+		return "on"
+	case triStateOff:
+		return "off"
+	default:
+		return "model-default"
+	}
+}
 
 // vllm_status (vllm.h).
 const (
 	vllmOK = 0
 )
 
-// cModelParams mirrors vllm_model_params.
+// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the
+// interior needs no padding on LP64, but the struct is 8-aligned (it holds
+// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets
+// and total size are asserted in vllmcpp_test.go.
 type cModelParams struct {
 	ModelPath           uintptr // const char*
-	TokenizerConfigPath uintptr // const char*
+	TokenizerConfigPath uintptr // const char*; NULL = <model_dir>/... (ABI v9)
 	BlockSize           int32
 	NumBlocks           int32
 	MaxModelLen         int32
 	MaxNumSeqs          int32
 	ToolParser          uintptr // const char*; NULL = auto-detect (ABI v4)
 	ReasoningParser     uintptr // const char*; NULL = auto-detect (ABI v5)
+	SpeculativeConfig   uintptr // const char* JSON; NULL = no speculation (ABI v6)
+	EnablePrefixCaching int32   // tri-state 0/1/2 (ABI v7)
+	MaxNumBatchedTokens int32   // <= 0 = per-arch default (ABI v9)
+	SchedulingPolicy    uintptr // const char*; NULL = "fcfs" (ABI v9)
+	KVTransferConfig    uintptr // const char* JSON; NULL = no connector (ABI v9)
+	EnableJumpForward   int32   // tri-state 0/1/2 (ABI v10)
+	_                   [4]byte // trailing pad to the struct's 8-byte alignment
 }
 
 // cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields
@@ -65,6 +98,12 @@ type cSamplingParams struct {
 	StructuredGrammar    uintptr // const char*
 	StructuredJSONObject int32
 	_                    [4]byte
+	// ABI v8 tail. LocalAI installs no custom logits processor, but the fields
+	// MUST be mirrored: the C side reads them off the pointer we hand it, so a
+	// Go struct that stopped at StructuredJSONObject would have the engine read
+	// 16 bytes past our allocation and call whatever garbage sat there.
+	LogitsProcessor         uintptr // vllm_logits_processor; NULL = none
+	LogitsProcessorUserData uintptr // void*
 }
 
 // cCompletion mirrors vllm_completion.

backend/go/vllm-cpp/options.go

@@ -1,30 +1,80 @@
 package main
 
-// Engine-sizing knobs carried through the model config's free-form
-// `options:` list ("key:value" entries), mirroring how the other in-house
-// backends pass engine-specific settings that have no proto field.
+// Load-time engine configuration, from two config surfaces:
+//
+//   - `engine_args:` (ModelOptions.EngineArgs, a JSON object) is the canonical
+//     one. Keys are spelled exactly as vLLM's own CLI flags, so a config written
+//     against vLLM works verbatim here - `speculative_config` and
+//     `kv_transfer_config` in particular take the same JSON documents vLLM's
+//     --speculative-config / --kv-transfer-config accept, and are handed to the
+//     engine unparsed.
+//   - `options:` (the free-form "key:value" list) is the older surface this
+//     backend shipped with. It is still honoured so existing configs keep
+//     working; engine_args wins on any key set in both.
+//
+// Anything unrecognised is ignored rather than fatal: the engine validates the
+// documents it is given and reports a precise error at load, and a config that
+// also carries knobs for a different backend must not fail the load here.
 
 import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/xlog"
 )
 
 type loadOptions struct {
 	blockSize  int32 // KV block size (tokens/block); engine default 32.
 	numBlocks  int32 // KV blocks to allocate; engine default 256.
 	maxNumSeqs int32 // max concurrent sequences; engine default 8.
+	// Max sequence length. Also settable through the model config's
+	// context_size / max_model_len; see Load for the precedence.
+	maxModelLen int32
+	// Per-step chunked-prefill token budget (ABI v9). 0 = the engine's
+	// bounded per-arch default.
+	maxNumBatchedTokens int32
+	// Automatic prefix caching tri-state (ABI v7): 0 = the model-capability
+	// default, 1 = force on, 2 = force off.
+	enablePrefixCaching int32
+	// Jump-forward decoding tri-state (ABI v10), SGLang's grammar-speed subset:
+	// 0 = defer to the environment (VT_ENABLE_JUMP_FORWARD, default off),
+	// 1 = force on, 2 = force off.
+	enableJumpForward int32
+	// Scheduler admission policy (ABI v9): "" = fcfs, else fcfs|priority|lpm.
+	schedulingPolicy string
 	// Engine-side parser selection (ABI v4/v5). Empty = the engine
 	// auto-detects from the chat template; "none" disables the reasoning
 	// split; unknown names fail the first chat call.
 	toolParser      string
 	reasoningParser string
+	// Speculative decoding (ABI v6), as vLLM's --speculative-config JSON:
+	// {"method":"mtp"|"dflash"|"ngram", ...}. Empty = no speculation.
+	speculativeConfig string
+	// External KV connector / LMCache (ABI v9), as vLLM's --kv-transfer-config
+	// JSON. Empty = no connector.
+	kvTransferConfig string
+	// Override for the tokenizer_config.json the chat template is read from
+	// (ABI v9). Empty = <model_dir>/tokenizer_config.json.
+	tokenizerConfigPath string
 }
 
 func parseOptions(opts *pb.ModelOptions) loadOptions {
 	lo := loadOptions{}
-	for _, o := range opts.GetOptions() {
+	applyOptionsList(&lo, opts.GetOptions())
+	applyEngineArgs(&lo, opts.GetEngineArgs())
+	return lo
+}
+
+// applyOptionsList reads the legacy free-form "key:value" list. strings.Cut
+// splits on the FIRST colon only, so a JSON object value survives intact.
+func applyOptionsList(lo *loadOptions, options []string) {
+	for _, o := range options {
 		k, v, found := strings.Cut(o, ":")
 		if !found {
 			continue
@@ -36,13 +86,211 @@ func parseOptions(opts *pb.ModelOptions) loadOptions {
 			lo.numBlocks = parseInt32(v, lo.numBlocks)
 		case "max_num_seqs":
 			lo.maxNumSeqs = parseInt32(v, lo.maxNumSeqs)
-		case "tool_parser":
+		case "max_num_batched_tokens":
+			lo.maxNumBatchedTokens = parseInt32(v, lo.maxNumBatchedTokens)
+		case "max_model_len":
+			lo.maxModelLen = parseInt32(v, lo.maxModelLen)
+		case "scheduling_policy", "schedule_policy":
+			lo.schedulingPolicy = strings.TrimSpace(v)
+		case "tool_parser", "tool_call_parser":
 			lo.toolParser = strings.TrimSpace(v)
 		case "reasoning_parser":
 			lo.reasoningParser = strings.TrimSpace(v)
+		case "speculative_config":
+			lo.speculativeConfig = strings.TrimSpace(v)
+		case "kv_transfer_config":
+			lo.kvTransferConfig = strings.TrimSpace(v)
+		case "tokenizer_config", "tokenizer_config_path":
+			lo.tokenizerConfigPath = strings.TrimSpace(v)
+		case "enable_prefix_caching", "enable_radix_attention":
+			if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
+				lo.enablePrefixCaching = boolTriState(b)
+			}
+		case "enable_jump_forward":
+			if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
+				lo.enableJumpForward = boolTriState(b)
+			}
 		}
 	}
-	return lo
+}
+
+// applyEngineArgs overlays the `engine_args:` JSON object. A document that does
+// not parse is logged and skipped: engine_args is shared with the other engines
+// (the vLLM and SGLang backends read the same field), so a stray key must not
+// take the model down.
+func applyEngineArgs(lo *loadOptions, engineArgs string) {
+	if strings.TrimSpace(engineArgs) == "" {
+		return
+	}
+	var args map[string]any
+	if err := json.Unmarshal([]byte(engineArgs), &args); err != nil {
+		xlog.Warn("[vllm-cpp] ignoring unparseable engine_args", "error", err)
+		return
+	}
+	for k, v := range args {
+		switch k {
+		case "block_size":
+			lo.blockSize = jsonInt32(v, lo.blockSize)
+		case "num_blocks":
+			lo.numBlocks = jsonInt32(v, lo.numBlocks)
+		case "max_num_seqs":
+			lo.maxNumSeqs = jsonInt32(v, lo.maxNumSeqs)
+		case "max_num_batched_tokens":
+			lo.maxNumBatchedTokens = jsonInt32(v, lo.maxNumBatchedTokens)
+		case "max_model_len":
+			lo.maxModelLen = jsonInt32(v, lo.maxModelLen)
+		case "scheduling_policy", "schedule_policy":
+			lo.schedulingPolicy = jsonString(v, lo.schedulingPolicy)
+		case "tool_parser", "tool_call_parser":
+			lo.toolParser = jsonString(v, lo.toolParser)
+		case "reasoning_parser":
+			lo.reasoningParser = jsonString(v, lo.reasoningParser)
+		case "tokenizer_config", "tokenizer_config_path":
+			lo.tokenizerConfigPath = jsonString(v, lo.tokenizerConfigPath)
+		case "speculative_config":
+			lo.speculativeConfig = jsonDocument(v, lo.speculativeConfig, k)
+		case "kv_transfer_config":
+			lo.kvTransferConfig = jsonDocument(v, lo.kvTransferConfig, k)
+		case "enable_prefix_caching", "enable_radix_attention":
+			if b, ok := v.(bool); ok {
+				lo.enablePrefixCaching = boolTriState(b)
+			}
+		case "enable_jump_forward":
+			if b, ok := v.(bool); ok {
+				lo.enableJumpForward = boolTriState(b)
+			}
+		default:
+			xlog.Debug("[vllm-cpp] ignoring unknown engine_args key", "key", k)
+		}
+	}
+}
+
+// boolTriState maps a YAML/JSON boolean onto the ABI's tri-state encoding. An
+// explicit `false` must reach the engine as force-OFF (2), NOT as the 0 that
+// means "defer". The difference is real in both directions: prefix caching
+// defaults ON for dense archs and OFF for hybrid ones, and jump forward defers
+// to VT_ENABLE_JUMP_FORWARD.
+func boolTriState(on bool) int32 {
+	if on {
+		return triStateOn
+	}
+	return triStateOff
+}
+
+// jsonDocument normalises an object-valued engine_args entry to a JSON string
+// for the C ABI. YAML nesting arrives as a map (the natural spelling); a
+// pre-encoded JSON string is accepted too, since a config round-tripped through
+// a flat store may carry it that way.
+func jsonDocument(v any, fallback string, key string) string {
+	switch t := v.(type) {
+	case string:
+		if strings.TrimSpace(t) == "" {
+			return fallback
+		}
+		return t
+	default:
+		buf, err := json.Marshal(t)
+		if err != nil {
+			xlog.Warn("[vllm-cpp] ignoring unencodable engine_args value", "key", key, "error", err)
+			return fallback
+		}
+		return string(buf)
+	}
+}
+
+func jsonString(v any, fallback string) string {
+	s, ok := v.(string)
+	if !ok {
+		return fallback
+	}
+	return strings.TrimSpace(s)
+}
+
+// jsonInt32 accepts the float64 a JSON number decodes to, plus the string
+// spelling a YAML config may produce. Non-positive values keep the fallback:
+// every knob this covers uses "<= 0 means the engine default".
+func jsonInt32(v any, fallback int32) int32 {
+	switch t := v.(type) {
+	case float64:
+		if t <= 0 || t > 1<<31-1 {
+			return fallback
+		}
+		return int32(t)
+	case string:
+		return parseInt32(t, fallback)
+	default:
+		return fallback
+	}
+}
+
+// resolveDraftModelPath rewrites a DFlash draft reference into an absolute path
+// the engine can actually open.
+//
+// The engine resolves `speculative_config.model` against a directory containing
+// config.json, or against ~/.cache/huggingface/hub/models--<org>--<repo>/
+// snapshots/* - and it NEVER downloads. LocalAI keeps models in its own
+// directory, so a bare HF repo id (the spelling the vLLM docs teach) misses the
+// HF cache and dies deep in the load with "draft checkpoint not found", which
+// reads like a broken checkpoint rather than a missing download.
+//
+// So: try the reference as given, then the last path segment under the models
+// dir (`z-lab/Qwen3.6-27B-DFlash` -> `<models>/Qwen3.6-27B-DFlash`, which is
+// what LocalAI's own downloader produces), then the whole reference under the
+// models dir. If none exist, fail HERE with a message naming both what was
+// asked for and where we looked.
+//
+// mtp and ngram carry no separate draft checkpoint, so they pass through. A
+// document that does not parse also passes through: the engine owns config
+// validation and produces the better error.
+func resolveDraftModelPath(speculativeConfig, modelsDir string) (string, error) {
+	if strings.TrimSpace(speculativeConfig) == "" {
+		return speculativeConfig, nil
+	}
+	var spec map[string]any
+	if err := json.Unmarshal([]byte(speculativeConfig), &spec); err != nil {
+		return speculativeConfig, nil
+	}
+	if method, _ := spec["method"].(string); !strings.EqualFold(method, "dflash") {
+		return speculativeConfig, nil
+	}
+
+	ref, _ := spec["model"].(string)
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return "", fmt.Errorf(
+			"vllm-cpp: speculative_config method %q requires a \"model\" key naming the draft checkpoint", "dflash")
+	}
+
+	candidates := []string{ref}
+	if modelsDir != "" {
+		if base := path.Base(filepath.ToSlash(ref)); base != "" && base != "." && base != "/" {
+			candidates = append(candidates, filepath.Join(modelsDir, base))
+		}
+		candidates = append(candidates, filepath.Join(modelsDir, filepath.FromSlash(ref)))
+	}
+
+	for _, c := range candidates {
+		if _, err := os.Stat(filepath.Join(c, "config.json")); err != nil {
+			continue
+		}
+		abs, err := filepath.Abs(c)
+		if err != nil {
+			abs = c
+		}
+		spec["model"] = abs
+		out, err := json.Marshal(spec)
+		if err != nil {
+			return "", fmt.Errorf("vllm-cpp: re-encoding speculative_config: %w", err)
+		}
+		xlog.Info("[vllm-cpp] resolved DFlash draft checkpoint", "reference", ref, "path", abs)
+		return string(out), nil
+	}
+
+	return "", fmt.Errorf(
+		"vllm-cpp: DFlash draft checkpoint %q not found (looked in: %s). "+
+			"The engine does not download drafts - install the draft model into LocalAI first, "+
+			"or set speculative_config.model to an absolute path to a directory containing config.json",
+		ref, strings.Join(candidates, ", "))
 }
 
 func parseInt32(s string, fallback int32) int32 {

backend/go/vllm-cpp/package.sh

@@ -43,50 +43,6 @@ elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
     cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
 elif [ $(uname -s) = "Darwin" ]; then
     echo "Detected Darwin"
-    # Vendor the optional MLX GEMM provider, when libvllm was built against it.
-    # Three facts drive every line below, each verified on an Apple M4 before it
-    # was written:
-    #   1. libvllm.dylib carries an LC_LOAD_DYLIB on @rpath/libmlx.dylib, and its
-    #      build-time LC_RPATH points inside the build venv. That path does not
-    #      exist on a user's machine, so it must become @loader_path/lib.
-    #   2. MLX finds its ~100 MB mlx.metallib beside its OWN dylib, so the two
-    #      files have to land in the same directory or every Metal op dies with
-    #      "Failed to load the default metallib".
-    #   3. install_name_tool invalidates the code signature, and macOS refuses to
-    #      load an arm64 image whose signature does not match, so the patched
-    #      library must be re-signed ad-hoc afterwards.
-    if otool -L "$CURDIR/package/libvllm.dylib" 2>/dev/null | grep -q "libmlx.dylib"; then
-        MLX_LIB_DIR="${MLX_ROOT}/lib"
-        if [ ! -f "$MLX_LIB_DIR/libmlx.dylib" ] || [ ! -f "$MLX_LIB_DIR/mlx.metallib" ]; then
-            echo "Error: libvllm.dylib links libmlx.dylib but $MLX_LIB_DIR is missing libmlx.dylib/mlx.metallib" >&2
-            exit 1
-        fi
-        echo "Vendoring the MLX GEMM provider from $MLX_LIB_DIR"
-        cp -fLv "$MLX_LIB_DIR/libmlx.dylib" "$CURDIR/package/lib/"
-        cp -fLv "$MLX_LIB_DIR/mlx.metallib" "$CURDIR/package/lib/"
-        # MLX is MIT and we redistribute its binaries, so its license ships with
-        # them. mlx-metal is the wheel carrying the dylib and the metallib.
-        MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx_metal-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
-        if [ -z "$MLX_LICENSE" ]; then
-            MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
-        fi
-        if [ -z "$MLX_LICENSE" ]; then
-            echo "Error: could not find the MLX LICENSE to redistribute alongside libmlx.dylib" >&2
-            exit 1
-        fi
-        cp -fLv "$MLX_LICENSE" "$CURDIR/package/lib/LICENSE.mlx"
-        # Drop every build-tree rpath, then point at the packaged copy.
-        otool -l "$CURDIR/package/libvllm.dylib" | awk '/LC_RPATH/{f=1;next} f&&/ path /{print $2;f=0}' | while read -r rp; do
-            install_name_tool -delete_rpath "$rp" "$CURDIR/package/libvllm.dylib" 2>/dev/null || true
-        done
-        install_name_tool -add_rpath "@loader_path/lib" "$CURDIR/package/libvllm.dylib"
-        codesign -f -s - "$CURDIR/package/libvllm.dylib"
-        # A broken rpath must fail the BUILD, not the user's first inference.
-        if ! otool -l "$CURDIR/package/libvllm.dylib" | grep -q "@loader_path/lib"; then
-            echo "Error: libvllm.dylib did not get the @loader_path/lib rpath" >&2
-            exit 1
-        fi
-    fi
 else
     echo "Error: Could not detect architecture"
     exit 1

backend/go/vllm-cpp/vllmcpp_test.go

@@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) {
 	RunSpecs(t, "vllm-cpp suite")
 }
 
-// The Go POD mirrors must match the C struct layout of vllm.h (ABI v2)
+// The Go POD mirrors must match the C struct layout of vllm.h (ABI v9)
 // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
 // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
 var _ = Describe("C ABI struct mirrors", func() {
@@ -30,10 +30,18 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.MaxNumSeqs)).To(Equal(uintptr(28)))
 		Expect(unsafe.Offsetof(p.ToolParser)).To(Equal(uintptr(32)))
 		Expect(unsafe.Offsetof(p.ReasoningParser)).To(Equal(uintptr(40)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(48)))
+		Expect(unsafe.Offsetof(p.SpeculativeConfig)).To(Equal(uintptr(48)))
+		Expect(unsafe.Offsetof(p.EnablePrefixCaching)).To(Equal(uintptr(56)))
+		Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60)))
+		Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64)))
+		Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72)))
+		Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80)))
+		// 88, not 84: the struct is 8-aligned (it holds pointers), so the
+		// trailing int32 is padded out. Go pads identically.
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88)))
 	})
 
-	It("cSamplingParams matches vllm_sampling_params (ABI v2)", func() {
+	It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() {
 		var p cSamplingParams
 		Expect(unsafe.Offsetof(p.Temperature)).To(Equal(uintptr(0)))
 		Expect(unsafe.Offsetof(p.TopP)).To(Equal(uintptr(4)))
@@ -55,7 +63,9 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.NStructuredChoice)).To(Equal(uintptr(96)))
 		Expect(unsafe.Offsetof(p.StructuredGrammar)).To(Equal(uintptr(104)))
 		Expect(unsafe.Offsetof(p.StructuredJSONObject)).To(Equal(uintptr(112)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120)))
+		Expect(unsafe.Offsetof(p.LogitsProcessor)).To(Equal(uintptr(120)))
+		Expect(unsafe.Offsetof(p.LogitsProcessorUserData)).To(Equal(uintptr(128)))
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136)))
 	})
 
 	It("cCompletion matches vllm_completion", func() {
@@ -68,6 +78,23 @@ var _ = Describe("C ABI struct mirrors", func() {
 	})
 })
 
+// Pin/mirror skew is the failure mode this backend is most exposed to: the Go
+// PODs above are hand-written against one VLLM_ABI_VERSION, and the Makefile
+// pins the vllm.cpp commit that produces it. This spec catches drift without
+// needing model weights - set VLLM_CPP_LIBRARY to a built libvllm and it binds
+// every symbol and compares the library's reported ABI against the mirrors'.
+var _ = Describe("real library ABI handshake", func() {
+	It("binds every symbol and reports the ABI the mirrors were written against", func() {
+		lib := os.Getenv("VLLM_CPP_LIBRARY")
+		if lib == "" {
+			Skip("VLLM_CPP_LIBRARY not set; skipping the real-library handshake")
+		}
+		Expect(registerLib(lib)).To(Succeed())
+		Expect(vllmABIVersion()).To(Equal(int32(abiVersion)))
+		Expect(vllmVersion()).NotTo(BeEmpty())
+	})
+})
+
 var _ = Describe("parseOptions", func() {
 	It("extracts the engine sizing knobs", func() {
 		lo := parseOptions(&pb.ModelOptions{Options: []string{
@@ -83,6 +110,129 @@ var _ = Describe("parseOptions", func() {
 		}})
 		Expect(lo).To(Equal(loadOptions{}))
 	})
+
+	It("carries a speculative_config JSON value through the legacy options list", func() {
+		// strings.Cut splits on the FIRST colon only, so a JSON object value
+		// survives the "key:value" spelling intact.
+		lo := parseOptions(&pb.ModelOptions{Options: []string{
+			`speculative_config:{"method":"mtp","num_speculative_tokens":1}`,
+		}})
+		Expect(lo.speculativeConfig).To(Equal(`{"method":"mtp","num_speculative_tokens":1}`))
+	})
+})
+
+var _ = Describe("engine_args", func() {
+	It("maps every load knob onto the C model params", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
+			"block_size": 64,
+			"num_blocks": 1024,
+			"max_model_len": 16384,
+			"max_num_seqs": 32,
+			"max_num_batched_tokens": 8192,
+			"enable_prefix_caching": true,
+			"scheduling_policy": "lpm",
+			"tool_parser": "qwen3",
+			"reasoning_parser": "deepseek_r1",
+			"tokenizer_config": "/models/tok/tokenizer_config.json"
+		}`})
+		Expect(lo.blockSize).To(Equal(int32(64)))
+		Expect(lo.numBlocks).To(Equal(int32(1024)))
+		Expect(lo.maxModelLen).To(Equal(int32(16384)))
+		Expect(lo.maxNumSeqs).To(Equal(int32(32)))
+		Expect(lo.maxNumBatchedTokens).To(Equal(int32(8192)))
+		Expect(lo.enablePrefixCaching).To(Equal(int32(1)))
+		Expect(lo.schedulingPolicy).To(Equal("lpm"))
+		Expect(lo.toolParser).To(Equal("qwen3"))
+		Expect(lo.reasoningParser).To(Equal("deepseek_r1"))
+		Expect(lo.tokenizerConfigPath).To(Equal("/models/tok/tokenizer_config.json"))
+	})
+
+	It("re-marshals a nested speculative_config object to JSON for the engine", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
+			"speculative_config": {"method": "mtp", "num_speculative_tokens": 1}
+		}`})
+		Expect(lo.speculativeConfig).To(MatchJSON(`{"method":"mtp","num_speculative_tokens":1}`))
+	})
+
+	It("re-marshals a nested kv_transfer_config object (LMCache) to JSON", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
+			"kv_transfer_config": {
+				"kv_connector": "LMCacheConnector",
+				"kv_role": "kv_both",
+				"kv_connector_extra_config": {"host": "127.0.0.1", "port": 65432}
+			}
+		}`})
+		Expect(lo.kvTransferConfig).To(MatchJSON(`{
+			"kv_connector":"LMCacheConnector",
+			"kv_role":"kv_both",
+			"kv_connector_extra_config":{"host":"127.0.0.1","port":65432}
+		}`))
+	})
+
+	It("accepts a pre-encoded JSON string for the object-valued knobs", func() {
+		// A config written by hand (or round-tripped through a flat store) may
+		// carry the object as a string; both spellings reach the engine the same.
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
+			"speculative_config": "{\"method\":\"ngram\",\"num_speculative_tokens\":4}"
+		}`})
+		Expect(lo.speculativeConfig).To(MatchJSON(`{"method":"ngram","num_speculative_tokens":4}`))
+	})
+
+	It("maps enable_prefix_caching false onto the force-OFF tri-state", func() {
+		// The C ABI tri-state is 0=model default, 1=on, 2=off, so an explicit
+		// `false` must NOT collapse to the 0 that means "let the model decide".
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_prefix_caching": false}`})
+		Expect(lo.enablePrefixCaching).To(Equal(int32(2)))
+	})
+
+	It("leaves the prefix-caching tri-state at the model default when unset", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`})
+		Expect(lo.enablePrefixCaching).To(Equal(int32(0)))
+	})
+
+	It("accepts the radix-attention alias upstream documents for prefix caching", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_radix_attention": true}`})
+		Expect(lo.enablePrefixCaching).To(Equal(int32(1)))
+	})
+
+	It("maps enable_jump_forward onto its own tri-state", func() {
+		// ABI v10. Same tri-state shape as prefix caching, and the same trap:
+		// an explicit false must be force-OFF (2), not the 0 that defers to the
+		// environment.
+		on := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": true}`})
+		Expect(on.enableJumpForward).To(Equal(int32(1)))
+		off := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": false}`})
+		Expect(off.enableJumpForward).To(Equal(int32(2)))
+		unset := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`})
+		Expect(unset.enableJumpForward).To(Equal(int32(0)))
+	})
+
+	It("reads enable_jump_forward from the legacy options list too", func() {
+		lo := parseOptions(&pb.ModelOptions{Options: []string{"enable_jump_forward:true"}})
+		Expect(lo.enableJumpForward).To(Equal(int32(1)))
+	})
+
+	It("lets engine_args override the legacy options list", func() {
+		lo := parseOptions(&pb.ModelOptions{
+			Options:    []string{"max_num_seqs:8", "block_size:16"},
+			EngineArgs: `{"max_num_seqs": 64}`,
+		})
+		Expect(lo.maxNumSeqs).To(Equal(int32(64))) // engine_args wins
+		Expect(lo.blockSize).To(Equal(int32(16)))  // untouched keys survive
+	})
+
+	It("ignores malformed engine_args rather than failing the load", func() {
+		lo := parseOptions(&pb.ModelOptions{
+			Options:    []string{"max_num_seqs:8"},
+			EngineArgs: `{not json`,
+		})
+		Expect(lo.maxNumSeqs).To(Equal(int32(8)))
+	})
+
+	It("ignores unknown keys", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"gpu_memory_utilization": 0.9}`})
+		Expect(lo).To(Equal(loadOptions{}))
+	})
 })
 
 var _ = Describe("samplingFromPredict", func() {
@@ -135,6 +285,91 @@ var _ = Describe("samplingFromPredict", func() {
 	})
 })
 
+// The engine resolves speculative_config.model against a local directory or
+// ~/.cache/huggingface/hub ONLY - it never downloads. LocalAI keeps models in
+// its own directory, so a bare repo id would miss the HF cache and fail deep in
+// the load with a confusing "draft checkpoint not found". Resolve it here.
+var _ = Describe("resolveDraftModelPath", func() {
+	var modelsDir string
+
+	BeforeEach(func() {
+		modelsDir = GinkgoT().TempDir()
+	})
+
+	// draftDir creates a plausible draft checkpoint under models/.
+	draftDir := func(name string) string {
+		d := filepath.Join(modelsDir, name)
+		Expect(os.MkdirAll(d, 0o750)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(d, "config.json"), []byte("{}"), 0o600)).To(Succeed())
+		return d
+	}
+
+	It("rewrites a repo id to the matching directory in the models dir", func() {
+		want := draftDir("Qwen3.6-27B-DFlash")
+		spec := `{"method":"dflash","model":"z-lab/Qwen3.6-27B-DFlash"}`
+		out, err := resolveDraftModelPath(spec, modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(MatchJSON(`{"method":"dflash","model":"` + want + `"}`))
+	})
+
+	It("rewrites a models-dir-relative path", func() {
+		want := draftDir("drafts__dflash")
+		spec := `{"method":"dflash","model":"drafts__dflash"}`
+		out, err := resolveDraftModelPath(spec, modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(ContainSubstring(want))
+	})
+
+	It("leaves an absolute path that already resolves alone", func() {
+		abs := draftDir("elsewhere")
+		spec := `{"method":"dflash","model":"` + abs + `"}`
+		out, err := resolveDraftModelPath(spec, modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(MatchJSON(spec))
+	})
+
+	It("fails with an actionable error when the draft is nowhere on disk", func() {
+		// Silently passing the repo id through would surface as an HF-cache
+		// miss inside the engine, which reads as "your model is broken".
+		spec := `{"method":"dflash","model":"z-lab/Not-Downloaded"}`
+		_, err := resolveDraftModelPath(spec, modelsDir)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("z-lab/Not-Downloaded"))
+		Expect(err.Error()).To(ContainSubstring(modelsDir))
+	})
+
+	It("requires a model key for dflash", func() {
+		_, err := resolveDraftModelPath(`{"method":"dflash"}`, modelsDir)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("model"))
+	})
+
+	It("leaves mtp and ngram configs untouched", func() {
+		// Neither has a separate draft checkpoint to resolve.
+		for _, spec := range []string{
+			`{"method":"mtp"}`,
+			`{"method":"ngram","num_speculative_tokens":4}`,
+		} {
+			out, err := resolveDraftModelPath(spec, modelsDir)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(MatchJSON(spec))
+		}
+	})
+
+	It("passes a malformed document through for the engine to reject", func() {
+		// The engine owns config validation and produces the better message.
+		out, err := resolveDraftModelPath(`{not json`, modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(Equal(`{not json`))
+	})
+
+	It("is a no-op on an empty config", func() {
+		out, err := resolveDraftModelPath("", modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(BeEmpty())
+	})
+})
+
 var _ = Describe("validModelPath", func() {
 	It("accepts a .gguf file", func() {
 		dir := GinkgoT().TempDir()

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=306c88f4d1286aec1bf96e544632897886af5501
+WHISPER_CPP_VERSION?=64d57d3df5c8dacee098577257edcaa154bf5ef3
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/index.yaml

@@ -193,22 +193,12 @@
   alias: "vllm-cpp"
   license: apache-2.0
   description: |
-    ALPHA development builds. Try it, but llama-cpp stays the recommendation for
-    production use.
-
-    vllm.cpp is an Apache-2.0 C++20 inference engine maintained by the LocalAI team,
-    developed in its own repository and usable without LocalAI. It began as a port of
-    vLLM and keeps vLLM as its reference implementation, checking output against it and
-    benchmarking against it, while growing a featureset of its own. It implements vLLM's
-    V1 architecture (paged KV cache, continuous batching, prefix caching, scheduler,
-    sampler) on a portable tensor runtime with no Python, PyTorch or ggml at inference
-    time. It loads GGUF as well as Hugging Face safetensors, supports structured output
-    (JSON schema / regex / choice / GBNF grammar) enforced in-engine, ships speculative
-    decoding and KV offload, and runs on CPU, NVIDIA CUDA (Blackwell-family), Apple
-    Metal and Vulkan.
-
-    The project is expected to be renamed as it diverges further from vLLM; the new
-    name is still to be decided.
+    vllm.cpp is a from-scratch C++20 port of vLLM created and maintained by the LocalAI team.
+    It mirrors vLLM's V1 architecture (paged KV cache, continuous batching, prefix caching,
+    scheduler, sampler) on a portable tensor runtime with no Python, PyTorch or ggml at
+    inference time. It loads Hugging Face safetensors and GGUF checkpoints, supports
+    structured output (JSON schema / regex / choice / GBNF grammar) enforced in-engine,
+    and runs on CPU, NVIDIA CUDA (Blackwell-family), Apple Metal and Vulkan.
   urls:
     - https://github.com/mudler/vllm.cpp
   tags:

core/config/vllm_spec.go

@@ -0,0 +1,117 @@
+package config
+
+// Speculative-decoding auto-defaults for the vllm-cpp backend, the safetensors
+// counterpart of the GGUF/llama.cpp hook in mtp.go.
+//
+// The two engines detect and spell the same feature differently. llama.cpp
+// reads `<arch>.nextn_predict_layers` out of the GGUF header and takes
+// `spec_type:draft-mtp` in `options:`; vllm.cpp reads `mtp_num_hidden_layers`
+// out of the checkpoint's config.json and takes vLLM's own
+// `--speculative-config` JSON, which LocalAI carries in `engine_args`. The
+// engine resolves the draft depth and the default k itself, so the config only
+// has to name the method.
+
+import (
+	"encoding/json"
+
+	"github.com/mudler/xlog"
+)
+
+// hfSpecConfig is the subset of a HuggingFace config.json that decides whether
+// speculative decoding can be auto-enabled.
+type hfSpecConfig struct {
+	ModelType string `json:"model_type"`
+	// MtpNumHiddenLayers is the MTP head depth (upstream speculative.py reads
+	// it as n_predict for the qwen3_5 / qwen3_5_moe families).
+	MtpNumHiddenLayers uint32 `json:"mtp_num_hidden_layers"`
+	// DFlashConfig marks a z-lab DFlash DRAFT checkpoint (mask_token_id +
+	// target_layer_ids). Its presence means this repo is a draft, not a
+	// servable target.
+	DFlashConfig json.RawMessage `json:"dflash_config"`
+	// TextConfig is where multimodal checkpoints nest the language-model
+	// config, and therefore the MTP depth.
+	TextConfig *hfSpecConfig `json:"text_config"`
+}
+
+// parseHFSpecConfig decodes the speculative-relevant subset of a config.json.
+// A document that does not parse yields nothing rather than an error: detection
+// is best-effort and must never break an import.
+func parseHFSpecConfig(configJSON []byte) (hfSpecConfig, bool) {
+	if len(configJSON) == 0 {
+		return hfSpecConfig{}, false
+	}
+	var c hfSpecConfig
+	if err := json.Unmarshal(configJSON, &c); err != nil {
+		xlog.Debug("[vllm-spec] config.json did not parse; skipping detection", "error", err)
+		return hfSpecConfig{}, false
+	}
+	return c, true
+}
+
+// IsDFlashDraftConfig reports whether a HuggingFace config.json describes a
+// DFlash DRAFT checkpoint. Unlike MTP - whose head ships inside the target
+// checkpoint's `mtp.*` tensors - a DFlash draft is its own repo that can only
+// run paired with a target it verifies against, so it must never be configured
+// as a standalone model.
+func IsDFlashDraftConfig(configJSON []byte) bool {
+	c, ok := parseHFSpecConfig(configJSON)
+	if !ok {
+		return false
+	}
+	return len(c.DFlashConfig) > 0 ||
+		(c.TextConfig != nil && len(c.TextConfig.DFlashConfig) > 0)
+}
+
+// HasSafetensorsMTPHead reports whether a HuggingFace config.json declares a
+// self-speculating Multi-Token Prediction head, returning its depth. The depth
+// is informational: vllm.cpp resolves n_predict and the default
+// num_speculative_tokens from the checkpoint itself.
+//
+// DFlash drafts are excluded for the same reason `gemma4-assistant` GGUFs are
+// excluded from the llama.cpp hook: they carry head metadata but cannot
+// self-speculate.
+//
+// NOTE this is a safetensors-only signal. vllm.cpp rejects an MTP config over a
+// GGUF source, because the `mtp.*` draft tensors only exist in the safetensors
+// checkpoint - so the GGUF import path must not use this.
+func HasSafetensorsMTPHead(configJSON []byte) (uint32, bool) {
+	c, ok := parseHFSpecConfig(configJSON)
+	if !ok {
+		return 0, false
+	}
+	if IsDFlashDraftConfig(configJSON) {
+		return 0, false
+	}
+	n := c.MtpNumHiddenLayers
+	if n == 0 && c.TextConfig != nil {
+		n = c.TextConfig.MtpNumHiddenLayers
+	}
+	return n, n > 0
+}
+
+// ApplyVLLMSpeculativeDefaults enables MTP speculative decoding in cfg's
+// engine_args when nothing is configured there yet. It is a no-op when the user
+// already set a speculative_config, so an explicit choice (a different method,
+// an explicit k, a DFlash draft) is never clobbered.
+//
+// `layers` is the detected head depth and is only used for the diagnostic log
+// line - the engine derives the real k from the checkpoint.
+func ApplyVLLMSpeculativeDefaults(cfg *ModelConfig, layers uint32) {
+	if cfg == nil {
+		return
+	}
+	if _, set := cfg.EngineArgs["speculative_config"]; set {
+		xlog.Debug("[vllm-spec] MTP head detected but speculative_config already configured; leaving user choice intact",
+			"name", cfg.Name, "mtp_num_hidden_layers", layers)
+		return
+	}
+	if cfg.EngineArgs == nil {
+		cfg.EngineArgs = map[string]any{}
+	}
+	// Only the method: vllm.cpp defaults num_speculative_tokens to the
+	// checkpoint's own n_predict (speculative.py:865-875), which is the right
+	// value far more reliably than anything guessable here.
+	cfg.EngineArgs["speculative_config"] = map[string]any{"method": "mtp"}
+	xlog.Info("[vllm-spec] MTP head detected; enabling mtp speculative decoding",
+		"name", cfg.Name, "mtp_num_hidden_layers", layers)
+}

core/config/vllm_spec_test.go

@@ -0,0 +1,117 @@
+package config_test
+
+import (
+	. "github.com/mudler/LocalAI/core/config"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("vllm-cpp speculative-decoding auto-defaults", func() {
+	Context("HasSafetensorsMTPHead", func() {
+		It("detects a top-level mtp_num_hidden_layers", func() {
+			n, ok := HasSafetensorsMTPHead([]byte(`{
+				"model_type": "qwen3_5_moe",
+				"mtp_num_hidden_layers": 1
+			}`))
+			Expect(ok).To(BeTrue())
+			Expect(n).To(Equal(uint32(1)))
+		})
+
+		It("detects the head nested under text_config", func() {
+			// Multimodal checkpoints nest the language-model config, which is
+			// where the MTP depth lives (mirrors the engine's own resolution
+			// off config.raw text_config).
+			n, ok := HasSafetensorsMTPHead([]byte(`{
+				"model_type": "qwen3_5_moe",
+				"text_config": {"mtp_num_hidden_layers": 2}
+			}`))
+			Expect(ok).To(BeTrue())
+			Expect(n).To(Equal(uint32(2)))
+		})
+
+		It("reports no head when the key is absent", func() {
+			n, ok := HasSafetensorsMTPHead([]byte(`{"model_type": "llama"}`))
+			Expect(ok).To(BeFalse())
+			Expect(n).To(BeZero())
+		})
+
+		It("reports no head for a zero depth", func() {
+			_, ok := HasSafetensorsMTPHead([]byte(`{"mtp_num_hidden_layers": 0}`))
+			Expect(ok).To(BeFalse())
+		})
+
+		It("ignores a DFlash draft checkpoint", func() {
+			// A DFlash draft is a SEPARATE checkpoint that cannot serve alone:
+			// it needs a target to verify against. Same exclusion the GGUF path
+			// makes for gemma4-assistant drafts.
+			_, ok := HasSafetensorsMTPHead([]byte(`{
+				"model_type": "qwen3_dflash",
+				"mtp_num_hidden_layers": 1,
+				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0, 1]}
+			}`))
+			Expect(ok).To(BeFalse())
+		})
+
+		It("reports no head on unparseable JSON", func() {
+			_, ok := HasSafetensorsMTPHead([]byte(`{not json`))
+			Expect(ok).To(BeFalse())
+		})
+
+		It("reports no head on empty input", func() {
+			_, ok := HasSafetensorsMTPHead(nil)
+			Expect(ok).To(BeFalse())
+		})
+	})
+
+	Context("IsDFlashDraftConfig", func() {
+		It("recognises a draft by its dflash_config block", func() {
+			Expect(IsDFlashDraftConfig([]byte(`{
+				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0]}
+			}`))).To(BeTrue())
+		})
+
+		It("does not flag an ordinary checkpoint", func() {
+			Expect(IsDFlashDraftConfig([]byte(`{"model_type": "qwen3_5_moe"}`))).To(BeFalse())
+		})
+	})
+
+	Context("ApplyVLLMSpeculativeDefaults", func() {
+		It("writes the mtp method into engine_args", func() {
+			cfg := &ModelConfig{Name: "qwen"}
+			ApplyVLLMSpeculativeDefaults(cfg, 1)
+			Expect(cfg.EngineArgs).To(HaveKey("speculative_config"))
+			spec, ok := cfg.EngineArgs["speculative_config"].(map[string]any)
+			Expect(ok).To(BeTrue())
+			Expect(spec["method"]).To(Equal("mtp"))
+		})
+
+		It("leaves an existing speculative_config alone", func() {
+			cfg := &ModelConfig{
+				Name: "qwen",
+				LLMConfig: LLMConfig{
+					EngineArgs: map[string]any{
+						"speculative_config": map[string]any{"method": "ngram", "num_speculative_tokens": 4},
+					},
+				},
+			}
+			ApplyVLLMSpeculativeDefaults(cfg, 1)
+			spec := cfg.EngineArgs["speculative_config"].(map[string]any)
+			Expect(spec["method"]).To(Equal("ngram"))
+		})
+
+		It("preserves unrelated engine_args keys", func() {
+			cfg := &ModelConfig{
+				Name:      "qwen",
+				LLMConfig: LLMConfig{EngineArgs: map[string]any{"max_num_seqs": 32}},
+			}
+			ApplyVLLMSpeculativeDefaults(cfg, 1)
+			Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_seqs", 32))
+			Expect(cfg.EngineArgs).To(HaveKey("speculative_config"))
+		})
+
+		It("tolerates a nil config", func() {
+			Expect(func() { ApplyVLLMSpeculativeDefaults(nil, 1) }).ToNot(Panic())
+		})
+	})
+})

core/gallery/importers/llama-cpp.go

@@ -298,7 +298,15 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
 	// imported configs already carry spec_type:draft-mtp before the model is
 	// ever loaded - users see it in the YAML preview rather than discovering
 	// it after the first start.
-	maybeApplyMTPDefaults(&modelConfig, details, &cfg)
+	//
+	// vllm-cpp is excluded on both counts: `spec_type:*` are llama.cpp option
+	// keys it does not read, and vllm.cpp rejects an MTP config over a GGUF
+	// source outright (the `mtp.*` draft tensors exist only in the safetensors
+	// checkpoint). Its MTP auto-config runs in the vllm importer instead, over
+	// the safetensors config.json.
+	if backend != "vllm-cpp" {
+		maybeApplyMTPDefaults(&modelConfig, details, &cfg)
+	}
 
 	data, err := yaml.Marshal(modelConfig)
 	if err != nil {

core/gallery/importers/vllm.go

@@ -1,13 +1,21 @@
 package importers
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/gallery"
 	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/pkg/downloader"
+	"github.com/mudler/LocalAI/pkg/httpclient"
+	"github.com/mudler/xlog"
 	"go.yaml.in/yaml/v2"
 )
 
@@ -107,6 +115,12 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
 		// vllm python backend, so use_tokenizer_template carries over), but
 		// tool/reasoning parsing is the engine's own autoparser pipeline -
 		// the vllm-python tool_parser/reasoning_parser options don't apply.
+		//
+		// Auto-detect a Multi-Token Prediction head, the safetensors analogue
+		// of the llama-cpp importer's GGUF hook, so a freshly imported
+		// Qwen3.5 / Qwen3.6 config already carries speculative decoding in its
+		// engine_args instead of leaving the throughput on the table.
+		maybeApplyVLLMSpeculativeDefaults(&modelConfig, details)
 	} else {
 		// Auto-detect tool_parser and reasoning_parser for known model families.
 		// Surfacing them in the generated YAML lets users see and edit the choices.
@@ -132,3 +146,89 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
 		ConfigFile:  string(data),
 	}, nil
 }
+
+// maxSpecConfigProbeBytes caps the config.json body we read. Real ones are a
+// few KB; the cap keeps a hostile or mislabelled URL from streaming into the
+// importer.
+const maxSpecConfigProbeBytes = 1 << 20 // 1 MiB
+
+// specConfigProbeTimeout bounds the config.json fetch. Detection is an
+// optimisation, so it must never hold an import open for long.
+const specConfigProbeTimeout = 30 * time.Second
+
+// specConfigFetcher is the seam the config.json probe goes through, so tests can
+// drive the whole import path without a network round trip.
+var specConfigFetcher = fetchProbeBody
+
+// maybeApplyVLLMSpeculativeDefaults fetches the repository's config.json and,
+// when it declares a Multi-Token Prediction head, enables MTP speculative
+// decoding in the emitted engine_args. This is the safetensors counterpart of
+// the llama-cpp importer's GGUF header probe.
+//
+// Every failure is non-fatal and logged at debug: a network blip, a private
+// repo, or a config.json this doesn't understand must leave the import working
+// exactly as it did before, just without the speculative default.
+func maybeApplyVLLMSpeculativeDefaults(modelConfig *config.ModelConfig, details Details) {
+	probeURL := vllmSpecProbeURL(details)
+	if probeURL == "" {
+		return
+	}
+
+	body, err := specConfigFetcher(probeURL)
+	if err != nil {
+		xlog.Debug("[vllm-spec-importer] could not read config.json for MTP detection", "uri", probeURL, "error", err)
+		return
+	}
+
+	applySpecFromConfigJSON(modelConfig, body, details.URI)
+}
+
+// applySpecFromConfigJSON is the decision half of the probe, split out so it can
+// be exercised without a network round trip.
+func applySpecFromConfigJSON(modelConfig *config.ModelConfig, body []byte, uri string) {
+	if config.IsDFlashDraftConfig(body) {
+		// A DFlash draft cannot serve on its own - it only proposes tokens for
+		// a target model to verify. Say so rather than emitting a config that
+		// would fail at load.
+		xlog.Warn("[vllm-spec-importer] this repository is a DFlash DRAFT checkpoint, not a servable model; "+
+			"import the TARGET model and point engine_args.speculative_config at this repo "+
+			`({"method":"dflash","model":"<this repo>"})`, "uri", uri)
+		return
+	}
+
+	n, ok := config.HasSafetensorsMTPHead(body)
+	if !ok {
+		return
+	}
+	config.ApplyVLLMSpeculativeDefaults(modelConfig, n)
+}
+
+// vllmSpecProbeURL returns the HTTP(S) URL of the repository's config.json, or
+// "" when the import isn't backed by a HuggingFace repo we can fetch from (a
+// local directory import, an OCI artifact, ...).
+func vllmSpecProbeURL(details Details) string {
+	if details.HuggingFace == nil || details.HuggingFace.ModelID == "" {
+		return ""
+	}
+	return resolveHTTPProbe(downloader.HuggingFacePrefix + details.HuggingFace.ModelID + "/config.json")
+}
+
+// fetchProbeBody GETs a small remote JSON document under a short timeout.
+func fetchProbeBody(url string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), specConfigProbeTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := httpclient.NewWithTimeout(specConfigProbeTimeout).Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status %d", resp.StatusCode)
+	}
+	return io.ReadAll(io.LimitReader(resp.Body, maxSpecConfigProbeBytes))
+}

core/gallery/importers/vllm_spec_internal_test.go

@@ -0,0 +1,118 @@
+package importers
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/mudler/LocalAI/core/config"
+	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("vllm-cpp speculative auto-config (importer)", func() {
+	Context("applySpecFromConfigJSON", func() {
+		It("enables mtp when the checkpoint declares an MTP head", func() {
+			cfg := &config.ModelConfig{Name: "qwen3.5"}
+			applySpecFromConfigJSON(cfg, []byte(`{
+				"model_type": "qwen3_5_moe",
+				"mtp_num_hidden_layers": 1
+			}`), "huggingface://Qwen/Qwen3.5-A3B")
+			Expect(cfg.EngineArgs).To(HaveKeyWithValue("speculative_config",
+				map[string]any{"method": "mtp"}))
+		})
+
+		It("leaves a plain checkpoint untouched", func() {
+			cfg := &config.ModelConfig{Name: "llama"}
+			applySpecFromConfigJSON(cfg, []byte(`{"model_type": "llama"}`), "huggingface://meta/llama")
+			Expect(cfg.EngineArgs).To(BeEmpty())
+		})
+
+		It("refuses to configure a DFlash draft as a servable model", func() {
+			// The draft only proposes tokens; configuring it standalone would
+			// produce a model that cannot load.
+			cfg := &config.ModelConfig{Name: "dflash-draft"}
+			applySpecFromConfigJSON(cfg, []byte(`{
+				"model_type": "qwen3_dflash",
+				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0, 1]}
+			}`), "huggingface://z-lab/Qwen3.6-27B-DFlash")
+			Expect(cfg.EngineArgs).To(BeEmpty())
+		})
+
+		It("survives a config.json it cannot parse", func() {
+			cfg := &config.ModelConfig{Name: "weird"}
+			Expect(func() {
+				applySpecFromConfigJSON(cfg, []byte(`<html>404</html>`), "huggingface://a/b")
+			}).ToNot(Panic())
+			Expect(cfg.EngineArgs).To(BeEmpty())
+		})
+	})
+
+	Context("Import over a repository with an MTP head", func() {
+		var restore func()
+
+		BeforeEach(func() {
+			original := specConfigFetcher
+			restore = func() { specConfigFetcher = original }
+		})
+		AfterEach(func() { restore() })
+
+		importWith := func(backend, configJSON string) string {
+			specConfigFetcher = func(string) ([]byte, error) {
+				return []byte(configJSON), nil
+			}
+			importer := &VLLMImporter{}
+			out, err := importer.Import(Details{
+				URI:         "huggingface://Qwen/Qwen3.5-A3B",
+				Preferences: json.RawMessage(`{"backend": "` + backend + `"}`),
+				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
+			})
+			Expect(err).ToNot(HaveOccurred())
+			return out.ConfigFile
+		}
+
+		It("emits engine_args.speculative_config for vllm-cpp", func() {
+			yaml := importWith("vllm-cpp", `{"model_type":"qwen3_5_moe","mtp_num_hidden_layers":1}`)
+			Expect(yaml).To(ContainSubstring("engine_args:"))
+			Expect(yaml).To(ContainSubstring("speculative_config:"))
+			Expect(yaml).To(ContainSubstring("method: mtp"))
+		})
+
+		It("emits nothing speculative for the python vllm backend", func() {
+			// The python backend has its own speculative surface and its own
+			// version-dependent MTP support; this hook is vllm-cpp only.
+			yaml := importWith("vllm", `{"model_type":"qwen3_5_moe","mtp_num_hidden_layers":1}`)
+			Expect(yaml).NotTo(ContainSubstring("speculative_config"))
+		})
+
+		It("emits nothing speculative when the probe fails", func() {
+			specConfigFetcher = func(string) ([]byte, error) {
+				return nil, errors.New("network down")
+			}
+			importer := &VLLMImporter{}
+			out, err := importer.Import(Details{
+				URI:         "huggingface://Qwen/Qwen3.5-A3B",
+				Preferences: json.RawMessage(`{"backend": "vllm-cpp"}`),
+				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out.ConfigFile).NotTo(ContainSubstring("speculative_config"))
+		})
+	})
+
+	Context("vllmSpecProbeURL", func() {
+		It("resolves the repository's config.json to an HTTPS URL", func() {
+			url := vllmSpecProbeURL(Details{
+				URI:         "huggingface://Qwen/Qwen3.5-A3B",
+				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
+			})
+			Expect(url).To(ContainSubstring("Qwen/Qwen3.5-A3B"))
+			Expect(url).To(HaveSuffix("config.json"))
+			Expect(url).To(HavePrefix("https://"))
+		})
+
+		It("skips the probe when there is no HuggingFace repo behind the import", func() {
+			Expect(vllmSpecProbeURL(Details{URI: "/models/local-dir"})).To(BeEmpty())
+		})
+	})
+})

core/http/react-ui/package-lock.json

@@ -24,7 +24,7 @@
         "@modelcontextprotocol/sdk": "^1.30.0",
         "dompurify": "^3.4.12",
         "highlight.js": "^11.11.1",
-        "hono": "4.12.34",
+        "hono": "4.12.25",
         "i18next": "^26.0.8",
         "i18next-browser-languagedetector": "^8.2.1",
         "i18next-http-backend": "^3.0.6",
@@ -636,12 +636,12 @@
       }
     },
     "node_modules/@hono/node-server": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.0.tgz",
-      "integrity": "sha512-XovyyCCnBzW+zKu+z/zq8hwNs4KOR5rEMAOxo2f40Q5xoOI37IMm6MIg2COOUtUApo0i6850MTBKH2u4QLGIqg==",
+      "version": "1.19.14",
+      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz",
+      "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==",
       "license": "MIT",
       "engines": {
-        "node": ">=20"
+        "node": ">=18.14.1"
       },
       "peerDependencies": {
         "hono": "^4"
@@ -2879,9 +2879,9 @@
       "dev": true
     },
     "node_modules/fast-uri": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
-      "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz",
+      "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==",
       "funding": [
         {
           "type": "github",
@@ -3435,9 +3435,9 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.12.34",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.34.tgz",
-      "integrity": "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA==",
+      "version": "4.12.25",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.25.tgz",
+      "integrity": "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==",
       "license": "MIT",
       "engines": {
         "node": ">=16.9.0"
@@ -4196,9 +4196,9 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
     "node_modules/ip-address": {
-      "version": "10.4.0",
-      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz",
-      "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==",
+      "version": "10.2.0",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
+      "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==",
       "license": "MIT",
       "engines": {
         "node": ">= 12"
@@ -7138,9 +7138,9 @@
       }
     },
     "node_modules/undici": {
-      "version": "7.29.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
-      "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==",
+      "version": "7.28.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz",
+      "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==",
       "dev": true,
       "license": "MIT",
       "engines": {

core/http/react-ui/package.json

@@ -19,7 +19,7 @@
     "coverage:report": "nyc report"
   },
   "overrides": {
-    "hono": "4.12.34"
+    "hono": "4.12.25"
   },
   "dependencies": {
     "@codemirror/autocomplete": "^6.18.6",
@@ -38,7 +38,7 @@
     "@modelcontextprotocol/sdk": "^1.30.0",
     "dompurify": "^3.4.12",
     "highlight.js": "^11.11.1",
-    "hono": "4.12.34",
+    "hono": "4.12.25",
     "i18next": "^26.0.8",
     "i18next-browser-languagedetector": "^8.2.1",
     "i18next-http-backend": "^3.0.6",

core/services/advisorylock/leader_loop_test.go

@@ -54,57 +54,62 @@ var _ = Describe("RunLeaderLoop", func() {
 				close(done)
 			}()
 
-			Eventually(func() int32 {
-				return atomic.LoadInt32(&callCount)
-			}, 500*time.Millisecond, 10*time.Millisecond).Should(BeNumerically(">=", 1))
+			// Let it run a bit then cancel
+			time.Sleep(150 * time.Millisecond)
 			cancel()
 
+			// RunLeaderLoop should return
 			Eventually(done, 500*time.Millisecond).Should(BeClosed())
+
+			// Record count after cancellation
+			countAfterCancel := atomic.LoadInt32(&callCount)
+			time.Sleep(150 * time.Millisecond)
+			countLater := atomic.LoadInt32(&callCount)
+
+			Expect(countLater).To(Equal(countAfterCancel),
+				"function should stop being called after context cancellation")
 		})
 
 		It("only one leader executes at a time (two concurrent loops)", func() {
 			db := testutil.SetupTestDB()
 			const lockKey int64 = 5002
 
-			var running int32
-			entered := make(chan struct{}, 2)
-			release := make(chan struct{})
-			var releaseOnce sync.Once
+			var (
+				mu         sync.Mutex
+				maxRunning int32
+				running    int32
+			)
 
 			ctx, cancel := context.WithCancel(context.Background())
-			done := make(chan struct{}, 2)
-			DeferCleanup(func() {
-				cancel()
-				releaseOnce.Do(func() { close(release) })
-			})
+			defer cancel()
 
 			fn := func() {
-				atomic.AddInt32(&running, 1)
-				select {
-				case entered <- struct{}{}:
-				default:
+				cur := atomic.AddInt32(&running, 1)
+				mu.Lock()
+				if cur > maxRunning {
+					maxRunning = cur
 				}
-				<-release
+				mu.Unlock()
+
+				time.Sleep(30 * time.Millisecond)
+
 				atomic.AddInt32(&running, -1)
 			}
 
-			for range 2 {
-				go func() {
-					RunLeaderLoop(ctx, db, lockKey, 1*time.Millisecond, fn)
-					done <- struct{}{}
-				}()
-			}
-
-			Eventually(entered, 500*time.Millisecond).Should(Receive())
-			Consistently(func() int32 {
-				return atomic.LoadInt32(&running)
-			}, 50*time.Millisecond, 5*time.Millisecond).Should(Equal(int32(1)),
-				"expected only the lock holder to run while both loops tick")
+			// Start two competing leader loops with the same lock key
+			go RunLeaderLoop(ctx, db, lockKey, 50*time.Millisecond, fn)
+			go RunLeaderLoop(ctx, db, lockKey, 50*time.Millisecond, fn)
 
+			// Let them run for a while
+			time.Sleep(400 * time.Millisecond)
 			cancel()
-			releaseOnce.Do(func() { close(release) })
-			Eventually(done, 500*time.Millisecond).Should(Receive())
-			Eventually(done, 500*time.Millisecond).Should(Receive())
+
+			mu.Lock()
+			observed := maxRunning
+			mu.Unlock()
+
+			Expect(observed).To(BeNumerically("<=", 1),
+				"expected at most 1 goroutine running the leader function at a time")
 		})
 	})
 })

docs/content/features/text-generation.md

@@ -918,6 +918,200 @@ options:
 The full list of registered parsers lives in `sglang.srt.function_call`
 and `sglang.srt.parser.reasoning_parser`.
 
+### vllm.cpp
+
+[vllm.cpp](https://github.com/mudler/vllm.cpp) is the LocalAI team's C++ port of
+vLLM: the same continuous-batching scheduler, paged KV cache and prefix caching,
+with no Python at inference time. It consumes either a HuggingFace safetensors
+model directory or a `.gguf` file, and applies the model's chat template,
+tool-call parsing and reasoning split engine-side.
+
+#### Setup
+
+```yaml
+name: vllm-cpp
+backend: vllm-cpp
+parameters:
+  model: "Qwen/Qwen3-4B"
+context_size: 8192
+template:
+  use_tokenizer_template: true
+```
+
+#### Configuring the engine with `engine_args`
+
+The same `engine_args:` map the vLLM and SGLang backends accept is honoured
+here, with keys spelled exactly as vLLM's own CLI flags - so a `speculative_config`
+or `kv_transfer_config` block written for vLLM works verbatim. Unknown keys are
+ignored rather than fatal; the engine validates the documents it is handed and
+reports a precise error at load.
+
+```yaml
+name: qwen35-a3b
+backend: vllm-cpp
+parameters:
+  model: "Qwen/Qwen3.5-A3B"
+context_size: 16384
+template:
+  use_tokenizer_template: true
+engine_args:
+  # KV cache sizing: num_blocks * block_size tokens of cache.
+  block_size: 32
+  num_blocks: 1024
+  # Concurrency and the per-step chunked-prefill token budget.
+  max_num_seqs: 32
+  max_num_batched_tokens: 8192
+  # Automatic prefix caching. Omit to keep the model's own default
+  # (on for dense models, off for hybrid / attention-free ones).
+  enable_prefix_caching: true
+  # Scheduler admission order: fcfs (default), priority, or lpm
+  # (cache-aware longest-prefix-match; needs prefix caching to have any effect).
+  scheduling_policy: lpm
+```
+
+| Key | Meaning | Default |
+|-----|---------|---------|
+| `block_size` | KV-cache block size, in tokens per block | 32 |
+| `num_blocks` | KV-cache blocks to allocate | 256 |
+| `max_model_len` | Max sequence length; also settable as `context_size` / `max_model_len` | model config |
+| `max_num_seqs` | Max concurrent sequences the scheduler admits | 8 |
+| `max_num_batched_tokens` | Per-step chunked-prefill token budget | per-arch (2048 dense, 4096/8192 MoE) |
+| `enable_prefix_caching` | Automatic prefix caching; `enable_radix_attention` is an accepted alias | model default |
+| `enable_jump_forward` | Jump-forward decoding, which emits grammar-forced tokens without a model step. Only affects constrained requests (`grammar`, JSON schema) | off |
+| `scheduling_policy` | `fcfs`, `priority`, or `lpm` | `fcfs` |
+| `tool_parser` / `reasoning_parser` | Force a parser instead of chat-template auto-detection | auto |
+| `tokenizer_config` | Override the `tokenizer_config.json` the chat template is read from | `<model_dir>/tokenizer_config.json` |
+| `speculative_config` | Speculative decoding (see below) | disabled |
+| `kv_transfer_config` | External KV connector / LMCache (see below) | none |
+
+Raising `max_num_batched_tokens` lets more prefill land in a single step, at the
+cost of decode latency for requests queued behind it. The default deliberately
+does not scale with `max_num_seqs`, which is what keeps a large concurrent
+prefill from blowing up the per-step activation on the hybrid architectures.
+
+`enable_prefix_caching` and `enable_jump_forward` are tri-state at the engine
+boundary: omitting the key defers to a default (the model's own capability for
+prefix caching, an environment variable for jump forward), while an explicit
+`false` forces the feature off. Those are genuinely different - prefix caching
+defaults *on* for dense models - so write the key only when you mean to override.
+
+#### Speculative decoding
+
+`speculative_config:` takes the same JSON object as vLLM's
+`--speculative-config`. Three methods are supported.
+
+> **Architecture limit.** At the current engine pin, `mtp` and `dflash` are
+> **Qwen3.5 / Qwen3.6 only**. The engine builds a widened speculative KV cache
+> directly for those families rather than through the model registry, so a
+> speculative config on any other architecture (Llama, GLM, Gemma, Mistral, ...)
+> will not work regardless of checkpoint format. `ngram` needs no draft weights
+> and is not subject to this limit.
+
+> **Format support.** `mtp` and `dflash` now work from a `.gguf` target as well
+> as safetensors. An MTP head is read from the GGUF's `nextn.*` tensors when the
+> file declares `<arch>.nextn_predict_layers`; a GGUF exported WITHOUT the head
+> (converted with `--no-mtp`, or predating llama.cpp's Qwen3.5 MTP support) is
+> refused at load naming that as the reason. A DFlash draft may itself be a
+> `dflash`-arch GGUF, and the target may be a GGUF too. `ngram` needs no draft
+> weights and works on any format.
+
+**MTP** (Multi-Token Prediction) uses a draft head shipped inside the target
+checkpoint's own `mtp.*` tensors, so there is no second model to download. It
+requires a **safetensors** checkpoint - the `mtp.*` tensors do not survive GGUF
+conversion, and an MTP config over a `.gguf` model is rejected at load.
+
+```yaml
+engine_args:
+  speculative_config:
+    method: mtp
+    # Optional; defaults to the checkpoint's own head depth, which is
+    # usually the right value. Must be a multiple of that depth.
+    num_speculative_tokens: 1
+```
+
+**DFlash** uses a separate block-diffusion drafter that proposes a whole block
+of tokens in one non-autoregressive forward pass. Unlike MTP, the draft is its
+own checkpoint, so `model:` is **required**:
+
+```yaml
+engine_args:
+  speculative_config:
+    method: dflash
+    model: z-lab/Qwen3.6-27B-DFlash
+    num_speculative_tokens: 4
+```
+
+The draft shares the *target's* `embed_tokens` and `lm_head`, so both must come
+from the same model family and the target must be safetensors.
+
+**The engine does not download the draft.** `model:` is resolved, in order,
+as a path as given, then as the last path segment under LocalAI's models
+directory (`z-lab/Qwen3.6-27B-DFlash` → `<models>/Qwen3.6-27B-DFlash`, which is
+what LocalAI's own downloader produces), then as the whole reference under the
+models directory. Install the draft into LocalAI first, or give an absolute path
+to a directory containing `config.json`. If none of those resolve, the load
+fails immediately naming every location that was tried, rather than reporting a
+missing checkpoint from inside the engine.
+
+**N-gram** needs no draft model at all - it proposes from the prompt's own
+suffix history. `num_speculative_tokens` is required:
+
+```yaml
+engine_args:
+  speculative_config:
+    method: ngram
+    num_speculative_tokens: 4
+    prompt_lookup_min: 5
+    prompt_lookup_max: 5
+```
+
+> **Auto-configuration on import.** When you import a safetensors repository
+> with `backend: vllm-cpp`, LocalAI reads the checkpoint's `config.json` and, if
+> it declares an MTP head (`mtp_num_hidden_layers`), writes
+> `speculative_config: {method: mtp}` into the generated `engine_args` for you.
+> An explicit `speculative_config` in your own config is never overwritten.
+> Importing a DFlash *draft* repository is refused with a warning: a drafter
+> cannot serve on its own, so import the target model and point
+> `speculative_config.model` at the draft.
+
+#### External KV cache with LMCache
+
+`kv_transfer_config:` takes vLLM's `--kv-transfer-config` JSON and selects an
+external KV-cache connector. The `lm://` LMCache client lets prefill KV be
+stored to and reloaded from a shared `lmcache.v1.server`, so a prefix computed
+by one replica does not have to be recomputed by the next:
+
+```yaml
+engine_args:
+  kv_transfer_config:
+    kv_connector: LMCacheConnector
+    kv_role: kv_both          # required whenever kv_connector is set
+    kv_connector_extra_config:
+      host: 127.0.0.1
+      port: 65432
+```
+
+`kv_role` is one of `kv_producer` (store only), `kv_consumer` (load only), or
+`kv_both`. An unregistered connector name, a missing role, or a malformed
+document fails the load with an explicit error rather than silently running
+without the cache.
+
+#### Legacy `options:` list
+
+Earlier versions configured this backend through the flat `options:` list, and
+those configs keep working. Every key in the table above is still read from
+there in `key:value` form, and `engine_args` wins on any key set in both:
+
+```yaml
+options:
+  - max_num_seqs:32
+  - enable_prefix_caching:true
+```
+
+New configs should prefer `engine_args:`, which is the only place the nested
+`speculative_config` / `kv_transfer_config` documents can be written naturally
+rather than as a single-line JSON string.
+
 ### Transformers
 
 [Transformers](https://huggingface.co/docs/transformers/index) is a State-of-the-art Machine Learning library for PyTorch, TensorFlow, and JAX.

gallery/index.yaml

@@ -785,18 +785,35 @@
 - name: "qwen3.6-35b-a3b-uncensored-genesis-hermes-v6"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
-    - https://huggingface.co/HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive
     - https://huggingface.co/LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V6-GGUF
   description: |
-    Qwen3.6-35B-A3B Uncensored Genesis Hermes V6 is LuffyTheFox's multimodal,
-    agentic derivative of HauhauCS's uncensored Qwen3.6-35B-A3B model. It
-    combines Genesis tensor calibration with Hermes function-calling data while
-    retaining the 35B mixture-of-experts architecture, roughly 3B active
-    parameters per token, and the native 262K-token context window.
+    # Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive
 
-    This entry installs the Q8_0 GGUF together with its F16 multimodal projector
-    for llama.cpp. The model card recommends Jinja chat templates and at least a
-    128K context for its thinking behavior. License: Apache-2.0.
+    > **Join the Discord** for updates, roadmaps, projects, or just to chat.
+
+    Qwen3.6-35B-A3B uncensored by HauhauCS. **0/465 Refusals.**
+
+    > **HuggingFace's "Hardware Compatibility" widget doesn't recognize K_P quants** — it may show fewer files than actually exist. Click **"View +X variants"** or go to **Files and versions** to see all available downloads.
+
+    ## About
+
+    No changes to datasets or capabilities. Fully functional, 100% of what the original authors intended - just without the refusals.
+
+    These are meant to be the best lossless uncensored models out there.
+
+    ## Aggressive Variant
+
+    Stronger uncensoring — model is fully unlocked and won't refuse prompts. May occasionally append short disclaimers (baked into base model training, not refusals) but full content is always generated.
+
+    For a more conservative uncensor that keeps some safety guardrails, check the Balanced variant when it's available.
+
+    ## Downloads
+
+    All quants generated with importance matrix (imatrix) for optimal quality preservation on abliterated weights.
+
+    ## What are K_P quants?
+
+    ...
   license: "apache-2.0"
   tags:
     - llm
@@ -1992,7 +2009,7 @@
   files:
     - filename: ds4flash.gguf
       uri: https://huggingface.co/unsloth/DeepSeek-V4-Flash-GGUF
-      sha256: ba1d64ad8d77038124839956b614db2e889daa1a4ddc83060bb06ccb5a1d7461
+      sha256: 1bfdafd1c288eb1b2bcb629ee9e1b7567dcf0abbe4d20995905a3c3465e9bd1e
 - name: "qwopus3.6-35b-a3b-coder-mtp"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -2614,83 +2631,6 @@
     - filename: llama-cpp/models/LFM2.5-1.2B-Instruct-GGUF/LFM2.5-1.2B-Instruct-Q4_K_M.gguf
       sha256: b1b3de114215d9507409a662a501a631095a479a419584e8a2ded6304b19b4f5
       uri: https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct-GGUF/resolve/main/LFM2.5-1.2B-Instruct-Q4_K_M.gguf
-- &lfm2-5-2-6b
-  name: "lfm2.5-2.6b"
-  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
-  urls:
-    - https://huggingface.co/LiquidAI/LFM2.5-2.6B
-    - https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF
-  description: |
-    LFM2.5-2.6B is LiquidAI's compact, text-only reasoning model for on-device
-    agentic workloads. It has 2.69B parameters, a 128K-token context window,
-    multilingual support, and post-training for tool use, instruction following,
-    data extraction, RAG, and multi-step agents. This entry uses the recommended
-    Q4_K_M GGUF quantization from LiquidAI's official repository.
-  license: "other"
-  tags:
-    - llm
-    - gguf
-    - reasoning
-    - cpu
-    - gpu
-  icon: https://cdn-uploads.huggingface.co/production/uploads/61b8e2ba285851687028d395/2b08LKpev0DNEk6DlnWkY.png
-  variants:
-    - model: lfm2.5-2.6b-q8
-  overrides:
-    backend: llama-cpp
-    context_size: 131072
-    function:
-      automatic_tool_parsing_fallback: true
-      grammar:
-        disable: true
-    known_usecases:
-      - chat
-      - completion
-    options:
-      - use_jinja:true
-    parameters:
-      model: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q4_K_M.gguf
-      repeat_penalty: 1.1
-      temperature: 0.1
-      top_k: 50
-    template:
-      use_tokenizer_template: true
-  files:
-    - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q4_K_M.gguf
-      sha256: 79fdf00351b46cf26f020aead28d01889886be87c55fa0eb907e6f9b00bfee14
-      uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q4_K_M.gguf
-- !!merge <<: *lfm2-5-2-6b
-  name: "lfm2.5-2.6b-q8"
-  description: |
-    LFM2.5-2.6B is LiquidAI's compact, text-only reasoning model for on-device
-    agentic workloads. It has 2.69B parameters, a 128K-token context window,
-    multilingual support, and post-training for tool use, instruction following,
-    data extraction, RAG, and multi-step agents. This entry uses the higher-quality
-    Q8_0 GGUF quantization from LiquidAI's official repository.
-  variants: null
-  overrides:
-    backend: llama-cpp
-    context_size: 131072
-    function:
-      automatic_tool_parsing_fallback: true
-      grammar:
-        disable: true
-    known_usecases:
-      - chat
-      - completion
-    options:
-      - use_jinja:true
-    parameters:
-      model: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf
-      repeat_penalty: 1.1
-      temperature: 0.1
-      top_k: 50
-    template:
-      use_tokenizer_template: true
-  files:
-    - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf
-      sha256: 36587fdf27bdfc69caf2637273679a0870ec155162161bde6fd16e8c70bdb757
-      uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q8_0.gguf
 - name: "qwopus3.6-27b-coder-compat-mtp"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:

scripts/build/llama-cpp-build-target_test.sh

@@ -29,11 +29,4 @@ assert_target arm64 "" llama-cpp-cpu-all
 assert_target amd64 sycl_f16 llama-cpp-fallback
 assert_target amd64 sycl_f32 llama-cpp-fallback
 
-# ROCm exhausts the same 6h budget through volume rather than a stall: hipcc
-# compiles ggml's HIP kernels once per AMDGPU target, eleven of them, and the
-# CPU variant matrix goes on top. 2h27m before it was added, killed at exactly
-# 6h00m on every run since.
-assert_target amd64 hipblas llama-cpp-fallback
-assert_target arm64 hipblas llama-cpp-fallback
-
 echo "PASS: llama.cpp build target preserves CPU variants where supported"

website/content/blog/what-landed-in-localai-4-8.md

@@ -1,14 +1,14 @@
 ---
 title: "What landed in LocalAI 4.8"
-date: 2026-08-04
+date: 2026-08-01
 author: "Ettore Di Giacinto"
 category: "Release"
 tags: ["release", "vllm.cpp", "audio.cpp", "3d", "gallery", "distributed", "performance"]
-summary: "A new inference engine, 3D generation, one backend that serves six audio endpoints, and a web interface 3.48x lighter. 374 pull requests in twenty-one days."
+summary: "A new inference engine, 3D generation, one backend that serves six audio endpoints, and a web interface 3.48x lighter. 321 pull requests in eighteen days."
 extracss: ["blog.css"]
 ---
 
-LocalAI 4.8.0 is out, after twenty-one days and 374 merged pull requests. There are three new things LocalAI can do, and a lot of repair work on things it already did.
+LocalAI 4.8.0 is out. It took eighteen days and 321 merged pull requests, and it pulls in two directions at once: three new things LocalAI can do that it could not do before, and a long list of places where it now does the old things without lying to you.
 
 The full notes list everything. This post covers the parts that change what you do day to day, with the pull request numbers so you can read the diffs.
 
@@ -55,35 +55,12 @@ Every surface can override the choice: `variant` on `POST /models/apply`, `local
 
 One gap worth knowing about: in distributed mode `InstallModel` resolves against the frontend rather than the worker that will serve the model, so a cluster with a small frontend and large workers selects conservatively. PRs [#10943](https://github.com/mudler/LocalAI/pull/10943), [#10983](https://github.com/mudler/LocalAI/pull/10983), [#10992](https://github.com/mudler/LocalAI/pull/10992), [#11027](https://github.com/mudler/LocalAI/pull/11027) and [#11139](https://github.com/mudler/LocalAI/pull/11139).
 
-## A new engine: vllm.cpp (alpha)
+## A new engine: vllm.cpp
 
-[vllm.cpp](https://github.com/mudler/vllm.cpp) is Apache-2.0 and maintained by the LocalAI team. We want it community-first rather than a LocalAI-only engine, so it lives in its own repository with its own docs, benchmark record and issue tracker, and it runs without LocalAI anywhere in the picture. It began as a C++20 port of vLLM. It ships here as the `vllm-cpp` backend ([#11100](https://github.com/mudler/LocalAI/pull/11100)). It implements vLLM's V1 architecture, so paged KV cache, continuous batching, prefix caching, scheduler and sampler, on a portable tensor runtime with no Python, no PyTorch and no ggml at inference. vLLM stays its reference implementation: correctness is checked by comparing output against it, and the benchmark scoreboard is kept against it.
-
-It has grown features vLLM does not have, which is most of the reason the port exists. It loads GGUF as well as safetensors, runs on CPU, Apple Metal and Vulkan alongside CUDA 12 and 13 and L4T for GB10, and ships speculative decoding and KV offload. Its benchmark page now measures against llama.cpp, MLX-LM and DwarfStar as well as vLLM, because on that hardware those are the engines it competes with. The project is expected to be renamed, with the new name still to be decided; it is drifting far enough that vllm.cpp will eventually mislead.
+[vllm.cpp](https://github.com/mudler/vllm.cpp) is a from-scratch C++20 port of vLLM, written and maintained by the LocalAI team under Apache-2.0, and it ships here as the `vllm-cpp` backend ([#11100](https://github.com/mudler/LocalAI/pull/11100)). It mirrors vLLM's V1 architecture, so paged KV cache, continuous batching, prefix caching, scheduler and sampler, on a portable tensor runtime with no Python, no PyTorch and no ggml at inference. It loads Hugging Face safetensors and GGUF, enforces structured output inside the engine (JSON schema, regex, choice, GBNF), and builds for CPU amd64 and arm64, CUDA 12 and 13 including Blackwell, L4T for GB10, Vulkan and Darwin Metal.
 
 Tool calling is at llama.cpp parity by construction, because chat deliberately reuses the same autoparser path: full minja chat templates, `tool_choice: auto` lowered to a lazy structural-tag decode constraint, 30 tool dialects, 7 reasoning parsers, and streamed `ChatDelta` and `ToolCallDelta`.
 
-Numbers from the project's own [scoreboard](https://github.com/mudler/vllm.cpp/blob/master/docs/BENCHMARKS.md), which calls ties ties and losses losses. Above 1.0 means vllm.cpp is ahead:
-
-<div class="tw">
-<table>
-<thead><tr><th>Reference</th><th>Workload</th><th>Result</th></tr></thead>
-<tbody>
-<tr><td>vLLM</td><td>Qwen3.6-27B NVFP4, GB10</td><td>1.045x at concurrency 1, 1.007x to 1.017x from c2 to c32, output token-for-token identical</td></tr>
-<tr><td>vLLM</td><td>Qwen3.6-35B-A3B NVFP4, GB10</td><td>1.010x at c16 and 1.013x at c32, behind from c1 to c8 (0.817x at c1)</td></tr>
-<tr><td>llama.cpp</td><td>Qwen3.5-2B GGUF, CPU aarch64</td><td>prefill 1.18x, decode a tie, memory parity</td></tr>
-<tr><td>MLX-LM</td><td>Qwen3-0.6B, Apple M4</td><td>97.6% of warm total, prefill ahead</td></tr>
-<tr><td>DwarfStar (ds4)</td><td>DeepSeek-V4-Flash IQ2_XXS, one DGX Spark</td><td>16.28 vs 16.33 tok/s decode, 0.997x, a parity result</td></tr>
-</tbody>
-</table>
-</div>
-
-The upstream page is careful about its own noise: on the 27B grid the run-to-run spread is 0.5% and c2 through c32 land between 0.7% and 1.7%, so it calls those five ties rather than wins. The concurrency-1 result is the one it stands behind.
-
-The DeepSeek-V4-Flash row is the one that shows how far this has moved from being a vLLM port. It runs DeepSeek-V4-Flash at roughly 2-bit (IQ2_XXS mixed, about 80 GB) on a single DGX Spark, decoding at 16.28 tok/s against DwarfStar's 16.33. At 300B+ total parameters even a 4-bit checkpoint is 156 GB or more, so a 2-bit GGUF is what fits inside the Spark's 119 GiB unified pool, and reading GGUF is what makes that possible.
-
-Speculative decoding is in similar shape: MTP on Qwen3.6-27B NVFP4 is token-identical to vLLM's MTP and about 4% faster at concurrency 1.
-
 Configuration is a normal backend install:
 
 ```yaml
@@ -96,24 +73,9 @@ options:
 - max_num_seqs:16      # also: block_size:<n>, num_blocks:<n>
 ```
 
-**Treat these as alpha development builds, not a released backend.** vllm.cpp is early, and shipping it in 4.8 is about getting it in front of people who want to try it, not about recommending it for anything you care about. `llama-cpp` stays the default for real use.
+The CPU path is verified end to end against `Qwen3.5-2B-UD-Q8_K_XL.gguf` with the full Ginkgo suite, covering blocking and streaming byte-parity, greedy determinism, stop words, GBNF-constrained generation, concurrent streams, reasoning split and both `required` and `auto` tool calls. The maturity statement from the release notes is worth repeating in full:
 
-The CPU path is verified end to end against `Qwen3.5-2B-UD-Q8_K_XL.gguf` with the full Ginkgo suite, covering blocking and streaming byte-parity, greedy determinism, stop words, GBNF-constrained generation, concurrent streams, reasoning split and both `required` and `auto` tool calls. The GPU images build and ship, but their runtime behavior has not been through that gate. No throughput comparison against upstream vLLM is claimed. Expect rough edges, and please report what breaks.
-
-On Apple Silicon the image now ships vllm.cpp's MLX GEMM provider ([#11137](https://github.com/mudler/LocalAI/pull/11137)). Upstream keeps it off by default because it adds about 124 MB, so we measured before turning it on. Qwen3-1.7B-bf16 on an M4, p=512 g=128, both arms toggled on one binary so a build difference cannot explain the gap:
-
-<div class="tw">
-<table>
-<thead><tr><th>Batch</th><th>MLX tok/s</th><th>native tok/s</th><th>speedup</th><th>MLX TTFT</th><th>native TTFT</th></tr></thead>
-<tbody>
-<tr><td>1</td><td>5.79</td><td>3.08</td><td><b>1.88x</b></td><td>3.32 s</td><td>7.68 s</td></tr>
-<tr><td>4</td><td>15.75</td><td>10.24</td><td><b>1.54x</b></td><td>9.63 s</td><td>18.77 s</td></tr>
-<tr><td>16</td><td>38.65</td><td>17.69</td><td><b>2.19x</b></td><td>18.33 s</td><td>54.48 s</td></tr>
-</tbody>
-</table>
-</div>
-
-Two reps, with rep spread reaching 9.4%, so treat the multipliers as +/-10%. Time to first token roughly halves across the range.
+> The GPU images build and ship, but their runtime behavior has not been through the same e2e gate yet. This is a first release of a young engine: no throughput comparison against upstream vLLM is claimed here, and `llama-cpp` remains the default recommendation for general use. Try it, and please report what breaks.
 
 <figure>
 <video src="/media/vllm-race.mp4" muted loop playsinline preload="none" data-lazy aria-label="vllm.cpp generating tokens"></video>
@@ -122,7 +84,7 @@ Two reps, with rep spread reaching 9.4%, so treat the multipliers as +/-10%. Tim
 
 ## LocalAI generates 3D models now
 
-3D generation is a new modality, so it had to be wired through the whole stack: a `Generate3D` RPC in `backend.proto`, a `FLAG_3D` capability so the loader knows which backends can serve it, and `POST /v1/3d/generations`.
+This is a new modality rather than a new backend under an existing one, so it goes through the whole stack: a `Generate3D` RPC in `backend.proto`, a `FLAG_3D` capability so the loader knows which backends can serve it, and `POST /v1/3d/generations`.
 
 The first engine behind it is `trellis2cpp`, an image-to-3D backend over TRELLIS.2. You give it an image, you get a GLB back. The web UI has a page for it with a native GLB viewer, so you can turn the result around in the browser instead of downloading it to find out whether it worked, history kept in IndexedDB so a reload does not lose your generations, and previewable print remeshing for output you actually intend to send to a printer ([#10979](https://github.com/mudler/LocalAI/pull/10979)).
 
@@ -133,7 +95,7 @@ The first engine behind it is `trellis2cpp`, an image-to-3D backend over TRELLIS
 
 ## One backend, six audio endpoints
 
-The usual shape for audio is one backend per model family, which means a process per capability and a config file for each. `audio-cpp` wraps [audio.cpp](https://github.com/0xShug0/audio.cpp), a multi-family ggml audio engine. One backend process serves several unrelated families through a single runtime vocabulary, and works out which family a checkpoint belongs to from the GGUF's own `audiocpp.model_spec.family` metadata key. There is nothing backend-specific to write in the model config.
+The usual shape for audio is one backend per model family, which means a process per capability and a config file for each. `audio-cpp` wraps [audio.cpp](https://github.com/0xShug0/audio.cpp), a multi-family ggml audio engine, and inverts that: one backend process serves several unrelated families through a single runtime vocabulary, and works out which family a checkpoint belongs to from the GGUF's own `audiocpp.model_spec.family` metadata key. There is nothing backend-specific to write in the model config.
 
 <div class="tw">
 <table>
@@ -168,7 +130,7 @@ The `bonsai` backend serves the 1-bit (Q1_0) and ternary (Q2_0) Bonsai quantizat
 
 ## The operations bar became a page
 
-The old operations bar rendered one row per in-flight operation above every page. Queue four model installs and a backend and it took most of the viewport, on every route, until the last one finished. It was doing two jobs at once. A global "something is happening" signal only needs one line, and the detail of what is happening needs a page of its own.
+The old operations bar rendered one row per in-flight operation above every page. Queue four model installs and a backend and it took most of the viewport, on every route, until the last one finished. Two things were conflated there: a global "something is happening" signal, which needs one line, and the detail of what is happening, which needs somewhere to put it.
 
 The strip is now one line, permanently, showing a failure first and otherwise the least-advanced running operation, with a `+N more` pill. Its `✕` hides the strip and no longer cancels anything. That is a deliberate behavior change worth knowing about before you click it out of habit: the same glyph used to cancel a 17 GB download in one row and dismiss a message in the next. Cancelling moved to the new page, behind a button that says so.
 
@@ -217,6 +179,6 @@ Valkey Search joins the vector store options as the `valkey-store` backend ([#11
 
 This is also the release where localai.io split in two: the project site at the root, and the documentation under `/docs/`. Every URL that was published before still resolves, through 214 generated redirect stubs, because GitHub Pages has no server-side rewrites to do it properly ([#11243](https://github.com/mudler/LocalAI/pull/11243)).
 
-Twenty-five people contributed to this release, eleven of them for the first time. The gallery went from 1,221 entries to 1,515.
+Twenty-four people contributed to this release, eleven of them for the first time. The gallery went from 1,221 entries to 1,505.
 
 To upgrade, pull `localai/localai:latest` or re-run the install script. The [full changelog](https://github.com/mudler/LocalAI/compare/v4.7.1...v4.8.0) has everything this post left out.