fix(vllm-cpp): pass Darwin warning flag to Objective-C

The Metal source that triggers Apple Clang's constant-folding diagnostic is compiled as Objective-C. Pass the targeted suppression through CMAKE_OBJC_FLAGS as well as the C++ language flags so vllm.cpp's -Werror no longer promotes it.

Assisted-by: Codex:gpt-5

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=5a8312ef7b8aa7cf14e9a24ac568cabd8725d68a
+AUDIO_CPP_VERSION?=4e3aea2fd99aeaa5924e71c51eb2793846045332
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=cb9147fd0d9c08a9a84eee5ac405a73f4e10e3e1
+IK_LLAMA_VERSION?=60389410a1ff01f9d37dcc6261db33b3183bdea2
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=fcb79282a6bc52e13d858026c42b24fb6e63c97a
+CRISPASR_VERSION?=fe3caf8e363b27572dbdd1a9d37083f25e6decda
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=9e1c9025ae61167a3335454d7cc0de6093c21845
+VLLM_CPP_VERSION?=a42b8187caff02c570c28e19e4dc2b1d7f55ed14
 
 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
 # server, examples and tests of the engine are never built here.
@@ -56,6 +56,12 @@ endif
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Darwin)
 	LIB=libvllm.dylib
+	# Apple Clang diagnoses a pair of constant-folded array bounds in the Metal
+	# build as a GNU extension. Disable that diagnostic for both Objective-C and
+	# C++ because vllm.cpp appends target-local -Werror after these global flags.
+	CMAKE_ARGS+=-DCMAKE_CXX_FLAGS=-Wno-gnu-folding-constant
+	CMAKE_ARGS+=-DCMAKE_OBJC_FLAGS=-Wno-gnu-folding-constant
+	CMAKE_ARGS+=-DCMAKE_OBJCXX_FLAGS=-Wno-gnu-folding-constant
 else
 	LIB=libvllm.so
 endif

backend/go/vllm-cpp/backend.go

@@ -109,6 +109,16 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 
 	v.opts = parseOptions(opts)
 
+	// A DFlash draft is a second checkpoint the engine opens by path, and the
+	// engine never downloads one. Resolve it against LocalAI's models directory
+	// now so a repo-id spelling works, and so a missing draft fails here with an
+	// actionable message rather than as an HF-cache miss inside the load.
+	resolvedSpec, err := resolveDraftModelPath(v.opts.speculativeConfig, opts.ModelPath)
+	if err != nil {
+		return err
+	}
+	v.opts.speculativeConfig = resolvedSpec
+
 	mp := defaultModelParams()
 	if v.opts.blockSize > 0 {
 		mp.BlockSize = v.opts.blockSize
@@ -116,34 +126,62 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 	if v.opts.numBlocks > 0 {
 		mp.NumBlocks = v.opts.numBlocks
 	}
+	// Sequence-length precedence, narrowest source last: context_size is the
+	// generic LocalAI knob every backend honours, max_model_len is the
+	// vLLM-specific one, and engine_args.max_model_len is the explicit
+	// vllm-cpp override.
 	if opts.ContextSize > 0 {
 		mp.MaxModelLen = opts.ContextSize
 	}
+	if opts.MaxModelLen > 0 {
+		mp.MaxModelLen = opts.MaxModelLen
+	}
+	if v.opts.maxModelLen > 0 {
+		mp.MaxModelLen = v.opts.maxModelLen
+	}
 	if v.opts.maxNumSeqs > 0 {
 		mp.MaxNumSeqs = v.opts.maxNumSeqs
 	}
+	if v.opts.maxNumBatchedTokens > 0 {
+		mp.MaxNumBatchedTokens = v.opts.maxNumBatchedTokens
+	}
+	mp.EnablePrefixCaching = v.opts.enablePrefixCaching
+	mp.EnableJumpForward = v.opts.enableJumpForward
 
+	// Every string below is borrowed by C for the duration of the load call
+	// only (the library copies what it keeps), so the backing slices just have
+	// to outlive vllmEngineLoad - hence the single KeepAlive after it.
 	modelC := cString(model)
 	mp.ModelPath = uintptr(unsafe.Pointer(&modelC[0])) // #nosec G103 -- borrowed by C for the load call only
-	var toolParserC, reasoningParserC []byte
-	if v.opts.toolParser != "" {
-		toolParserC = cString(v.opts.toolParser)
-		mp.ToolParser = uintptr(unsafe.Pointer(&toolParserC[0])) // #nosec G103 -- borrowed by C for the load call only
-	}
-	if v.opts.reasoningParser != "" {
-		reasoningParserC = cString(v.opts.reasoningParser)
-		mp.ReasoningParser = uintptr(unsafe.Pointer(&reasoningParserC[0])) // #nosec G103 -- borrowed by C for the load call only
+	keep := [][]byte{modelC}
+	setStr := func(dst *uintptr, s string) {
+		if s == "" {
+			return
+		}
+		b := cString(s)
+		keep = append(keep, b)
+		*dst = uintptr(unsafe.Pointer(&b[0])) // #nosec G103 -- borrowed by C for the load call only
 	}
+	setStr(&mp.ToolParser, v.opts.toolParser)
+	setStr(&mp.ReasoningParser, v.opts.reasoningParser)
+	setStr(&mp.SpeculativeConfig, v.opts.speculativeConfig)
+	setStr(&mp.KVTransferConfig, v.opts.kvTransferConfig)
+	setStr(&mp.SchedulingPolicy, v.opts.schedulingPolicy)
+	setStr(&mp.TokenizerConfigPath, v.opts.tokenizerConfigPath)
 
 	xlog.Info("[vllm-cpp] Load", "model", model, "engine", vllmVersion(),
 		"blockSize", mp.BlockSize, "numBlocks", mp.NumBlocks,
-		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs)
+		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs,
+		"maxNumBatchedTokens", mp.MaxNumBatchedTokens,
+		"prefixCaching", triStateName(mp.EnablePrefixCaching),
+		"jumpForward", triStateName(mp.EnableJumpForward),
+		"schedulingPolicy", v.opts.schedulingPolicy,
+		"speculativeConfig", v.opts.speculativeConfig,
+		"kvTransferConfig", v.opts.kvTransferConfig)
 
 	var engine uintptr
 	rc := vllmEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params
-	runtime.KeepAlive(modelC)
-	runtime.KeepAlive(toolParserC)
-	runtime.KeepAlive(reasoningParserC)
+	runtime.KeepAlive(keep)
 	if rc != vllmOK {
 		return fmt.Errorf("vllm-cpp: engine load failed: %s", vllmLastError())
 	}

backend/go/vllm-cpp/govllmcpp.go

@@ -1,6 +1,6 @@
 package main
 
-// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v2).
+// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10).
 //
 // The structs below are hand-mirrored PODs of the C declarations, with
 // explicit padding so the Go layout matches the C layout on linux/darwin
@@ -18,23 +18,56 @@ import (
 )
 
 // abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h).
-const abiVersion = 5
+const abiVersion = 10
+
+// The ABI's tri-state toggles (enable_prefix_caching ABI v7,
+// enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
+// "defer" - to the model capability for prefix caching, to the environment for
+// jump forward. Only 2 is an explicit off.
+const (
+	triStateDefer int32 = 0
+	triStateOn    int32 = 1
+	triStateOff   int32 = 2
+)
+
+// triStateName renders a tri-state for the load log line, where "0" would
+// otherwise read as "off" rather than "whatever the default resolves to".
+func triStateName(state int32) string {
+	switch state {
+	case triStateOn:
+		return "on"
+	case triStateOff:
+		return "off"
+	default:
+		return "model-default"
+	}
+}
 
 // vllm_status (vllm.h).
 const (
 	vllmOK = 0
 )
 
-// cModelParams mirrors vllm_model_params.
+// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the
+// interior needs no padding on LP64, but the struct is 8-aligned (it holds
+// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets
+// and total size are asserted in vllmcpp_test.go.
 type cModelParams struct {
 	ModelPath           uintptr // const char*
-	TokenizerConfigPath uintptr // const char*
+	TokenizerConfigPath uintptr // const char*; NULL = <model_dir>/... (ABI v9)
 	BlockSize           int32
 	NumBlocks           int32
 	MaxModelLen         int32
 	MaxNumSeqs          int32
 	ToolParser          uintptr // const char*; NULL = auto-detect (ABI v4)
 	ReasoningParser     uintptr // const char*; NULL = auto-detect (ABI v5)
+	SpeculativeConfig   uintptr // const char* JSON; NULL = no speculation (ABI v6)
+	EnablePrefixCaching int32   // tri-state 0/1/2 (ABI v7)
+	MaxNumBatchedTokens int32   // <= 0 = per-arch default (ABI v9)
+	SchedulingPolicy    uintptr // const char*; NULL = "fcfs" (ABI v9)
+	KVTransferConfig    uintptr // const char* JSON; NULL = no connector (ABI v9)
+	EnableJumpForward   int32   // tri-state 0/1/2 (ABI v10)
+	_                   [4]byte // trailing pad to the struct's 8-byte alignment
 }
 
 // cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields
@@ -65,6 +98,12 @@ type cSamplingParams struct {
 	StructuredGrammar    uintptr // const char*
 	StructuredJSONObject int32
 	_                    [4]byte
+	// ABI v8 tail. LocalAI installs no custom logits processor, but the fields
+	// MUST be mirrored: the C side reads them off the pointer we hand it, so a
+	// Go struct that stopped at StructuredJSONObject would have the engine read
+	// 16 bytes past our allocation and call whatever garbage sat there.
+	LogitsProcessor         uintptr // vllm_logits_processor; NULL = none
+	LogitsProcessorUserData uintptr // void*
 }
 
 // cCompletion mirrors vllm_completion.

backend/go/vllm-cpp/options.go

@@ -1,30 +1,80 @@
 package main
 
-// Engine-sizing knobs carried through the model config's free-form
-// `options:` list ("key:value" entries), mirroring how the other in-house
-// backends pass engine-specific settings that have no proto field.
+// Load-time engine configuration, from two config surfaces:
+//
+//   - `engine_args:` (ModelOptions.EngineArgs, a JSON object) is the canonical
+//     one. Keys are spelled exactly as vLLM's own CLI flags, so a config written
+//     against vLLM works verbatim here - `speculative_config` and
+//     `kv_transfer_config` in particular take the same JSON documents vLLM's
+//     --speculative-config / --kv-transfer-config accept, and are handed to the
+//     engine unparsed.
+//   - `options:` (the free-form "key:value" list) is the older surface this
+//     backend shipped with. It is still honoured so existing configs keep
+//     working; engine_args wins on any key set in both.
+//
+// Anything unrecognised is ignored rather than fatal: the engine validates the
+// documents it is given and reports a precise error at load, and a config that
+// also carries knobs for a different backend must not fail the load here.
 
 import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/xlog"
 )
 
 type loadOptions struct {
 	blockSize  int32 // KV block size (tokens/block); engine default 32.
 	numBlocks  int32 // KV blocks to allocate; engine default 256.
 	maxNumSeqs int32 // max concurrent sequences; engine default 8.
+	// Max sequence length. Also settable through the model config's
+	// context_size / max_model_len; see Load for the precedence.
+	maxModelLen int32
+	// Per-step chunked-prefill token budget (ABI v9). 0 = the engine's
+	// bounded per-arch default.
+	maxNumBatchedTokens int32
+	// Automatic prefix caching tri-state (ABI v7): 0 = the model-capability
+	// default, 1 = force on, 2 = force off.
+	enablePrefixCaching int32
+	// Jump-forward decoding tri-state (ABI v10), SGLang's grammar-speed subset:
+	// 0 = defer to the environment (VT_ENABLE_JUMP_FORWARD, default off),
+	// 1 = force on, 2 = force off.
+	enableJumpForward int32
+	// Scheduler admission policy (ABI v9): "" = fcfs, else fcfs|priority|lpm.
+	schedulingPolicy string
 	// Engine-side parser selection (ABI v4/v5). Empty = the engine
 	// auto-detects from the chat template; "none" disables the reasoning
 	// split; unknown names fail the first chat call.
 	toolParser      string
 	reasoningParser string
+	// Speculative decoding (ABI v6), as vLLM's --speculative-config JSON:
+	// {"method":"mtp"|"dflash"|"ngram", ...}. Empty = no speculation.
+	speculativeConfig string
+	// External KV connector / LMCache (ABI v9), as vLLM's --kv-transfer-config
+	// JSON. Empty = no connector.
+	kvTransferConfig string
+	// Override for the tokenizer_config.json the chat template is read from
+	// (ABI v9). Empty = <model_dir>/tokenizer_config.json.
+	tokenizerConfigPath string
 }
 
 func parseOptions(opts *pb.ModelOptions) loadOptions {
 	lo := loadOptions{}
-	for _, o := range opts.GetOptions() {
+	applyOptionsList(&lo, opts.GetOptions())
+	applyEngineArgs(&lo, opts.GetEngineArgs())
+	return lo
+}
+
+// applyOptionsList reads the legacy free-form "key:value" list. strings.Cut
+// splits on the FIRST colon only, so a JSON object value survives intact.
+func applyOptionsList(lo *loadOptions, options []string) {
+	for _, o := range options {
 		k, v, found := strings.Cut(o, ":")
 		if !found {
 			continue
@@ -36,13 +86,211 @@ func parseOptions(opts *pb.ModelOptions) loadOptions {
 			lo.numBlocks = parseInt32(v, lo.numBlocks)
 		case "max_num_seqs":
 			lo.maxNumSeqs = parseInt32(v, lo.maxNumSeqs)
-		case "tool_parser":
+		case "max_num_batched_tokens":
+			lo.maxNumBatchedTokens = parseInt32(v, lo.maxNumBatchedTokens)
+		case "max_model_len":
+			lo.maxModelLen = parseInt32(v, lo.maxModelLen)
+		case "scheduling_policy", "schedule_policy":
+			lo.schedulingPolicy = strings.TrimSpace(v)
+		case "tool_parser", "tool_call_parser":
 			lo.toolParser = strings.TrimSpace(v)
 		case "reasoning_parser":
 			lo.reasoningParser = strings.TrimSpace(v)
+		case "speculative_config":
+			lo.speculativeConfig = strings.TrimSpace(v)
+		case "kv_transfer_config":
+			lo.kvTransferConfig = strings.TrimSpace(v)
+		case "tokenizer_config", "tokenizer_config_path":
+			lo.tokenizerConfigPath = strings.TrimSpace(v)
+		case "enable_prefix_caching", "enable_radix_attention":
+			if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
+				lo.enablePrefixCaching = boolTriState(b)
+			}
+		case "enable_jump_forward":
+			if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
+				lo.enableJumpForward = boolTriState(b)
+			}
 		}
 	}
-	return lo
+}
+
+// applyEngineArgs overlays the `engine_args:` JSON object. A document that does
+// not parse is logged and skipped: engine_args is shared with the other engines
+// (the vLLM and SGLang backends read the same field), so a stray key must not
+// take the model down.
+func applyEngineArgs(lo *loadOptions, engineArgs string) {
+	if strings.TrimSpace(engineArgs) == "" {
+		return
+	}
+	var args map[string]any
+	if err := json.Unmarshal([]byte(engineArgs), &args); err != nil {
+		xlog.Warn("[vllm-cpp] ignoring unparseable engine_args", "error", err)
+		return
+	}
+	for k, v := range args {
+		switch k {
+		case "block_size":
+			lo.blockSize = jsonInt32(v, lo.blockSize)
+		case "num_blocks":
+			lo.numBlocks = jsonInt32(v, lo.numBlocks)
+		case "max_num_seqs":
+			lo.maxNumSeqs = jsonInt32(v, lo.maxNumSeqs)
+		case "max_num_batched_tokens":
+			lo.maxNumBatchedTokens = jsonInt32(v, lo.maxNumBatchedTokens)
+		case "max_model_len":
+			lo.maxModelLen = jsonInt32(v, lo.maxModelLen)
+		case "scheduling_policy", "schedule_policy":
+			lo.schedulingPolicy = jsonString(v, lo.schedulingPolicy)
+		case "tool_parser", "tool_call_parser":
+			lo.toolParser = jsonString(v, lo.toolParser)
+		case "reasoning_parser":
+			lo.reasoningParser = jsonString(v, lo.reasoningParser)
+		case "tokenizer_config", "tokenizer_config_path":
+			lo.tokenizerConfigPath = jsonString(v, lo.tokenizerConfigPath)
+		case "speculative_config":
+			lo.speculativeConfig = jsonDocument(v, lo.speculativeConfig, k)
+		case "kv_transfer_config":
+			lo.kvTransferConfig = jsonDocument(v, lo.kvTransferConfig, k)
+		case "enable_prefix_caching", "enable_radix_attention":
+			if b, ok := v.(bool); ok {
+				lo.enablePrefixCaching = boolTriState(b)
+			}
+		case "enable_jump_forward":
+			if b, ok := v.(bool); ok {
+				lo.enableJumpForward = boolTriState(b)
+			}
+		default:
+			xlog.Debug("[vllm-cpp] ignoring unknown engine_args key", "key", k)
+		}
+	}
+}
+
+// boolTriState maps a YAML/JSON boolean onto the ABI's tri-state encoding. An
+// explicit `false` must reach the engine as force-OFF (2), NOT as the 0 that
+// means "defer". The difference is real in both directions: prefix caching
+// defaults ON for dense archs and OFF for hybrid ones, and jump forward defers
+// to VT_ENABLE_JUMP_FORWARD.
+func boolTriState(on bool) int32 {
+	if on {
+		return triStateOn
+	}
+	return triStateOff
+}
+
+// jsonDocument normalises an object-valued engine_args entry to a JSON string
+// for the C ABI. YAML nesting arrives as a map (the natural spelling); a
+// pre-encoded JSON string is accepted too, since a config round-tripped through
+// a flat store may carry it that way.
+func jsonDocument(v any, fallback string, key string) string {
+	switch t := v.(type) {
+	case string:
+		if strings.TrimSpace(t) == "" {
+			return fallback
+		}
+		return t
+	default:
+		buf, err := json.Marshal(t)
+		if err != nil {
+			xlog.Warn("[vllm-cpp] ignoring unencodable engine_args value", "key", key, "error", err)
+			return fallback
+		}
+		return string(buf)
+	}
+}
+
+func jsonString(v any, fallback string) string {
+	s, ok := v.(string)
+	if !ok {
+		return fallback
+	}
+	return strings.TrimSpace(s)
+}
+
+// jsonInt32 accepts the float64 a JSON number decodes to, plus the string
+// spelling a YAML config may produce. Non-positive values keep the fallback:
+// every knob this covers uses "<= 0 means the engine default".
+func jsonInt32(v any, fallback int32) int32 {
+	switch t := v.(type) {
+	case float64:
+		if t <= 0 || t > 1<<31-1 {
+			return fallback
+		}
+		return int32(t)
+	case string:
+		return parseInt32(t, fallback)
+	default:
+		return fallback
+	}
+}
+
+// resolveDraftModelPath rewrites a DFlash draft reference into an absolute path
+// the engine can actually open.
+//
+// The engine resolves `speculative_config.model` against a directory containing
+// config.json, or against ~/.cache/huggingface/hub/models--<org>--<repo>/
+// snapshots/* - and it NEVER downloads. LocalAI keeps models in its own
+// directory, so a bare HF repo id (the spelling the vLLM docs teach) misses the
+// HF cache and dies deep in the load with "draft checkpoint not found", which
+// reads like a broken checkpoint rather than a missing download.
+//
+// So: try the reference as given, then the last path segment under the models
+// dir (`z-lab/Qwen3.6-27B-DFlash` -> `<models>/Qwen3.6-27B-DFlash`, which is
+// what LocalAI's own downloader produces), then the whole reference under the
+// models dir. If none exist, fail HERE with a message naming both what was
+// asked for and where we looked.
+//
+// mtp and ngram carry no separate draft checkpoint, so they pass through. A
+// document that does not parse also passes through: the engine owns config
+// validation and produces the better error.
+func resolveDraftModelPath(speculativeConfig, modelsDir string) (string, error) {
+	if strings.TrimSpace(speculativeConfig) == "" {
+		return speculativeConfig, nil
+	}
+	var spec map[string]any
+	if err := json.Unmarshal([]byte(speculativeConfig), &spec); err != nil {
+		return speculativeConfig, nil
+	}
+	if method, _ := spec["method"].(string); !strings.EqualFold(method, "dflash") {
+		return speculativeConfig, nil
+	}
+
+	ref, _ := spec["model"].(string)
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return "", fmt.Errorf(
+			"vllm-cpp: speculative_config method %q requires a \"model\" key naming the draft checkpoint", "dflash")
+	}
+
+	candidates := []string{ref}
+	if modelsDir != "" {
+		if base := path.Base(filepath.ToSlash(ref)); base != "" && base != "." && base != "/" {
+			candidates = append(candidates, filepath.Join(modelsDir, base))
+		}
+		candidates = append(candidates, filepath.Join(modelsDir, filepath.FromSlash(ref)))
+	}
+
+	for _, c := range candidates {
+		if _, err := os.Stat(filepath.Join(c, "config.json")); err != nil {
+			continue
+		}
+		abs, err := filepath.Abs(c)
+		if err != nil {
+			abs = c
+		}
+		spec["model"] = abs
+		out, err := json.Marshal(spec)
+		if err != nil {
+			return "", fmt.Errorf("vllm-cpp: re-encoding speculative_config: %w", err)
+		}
+		xlog.Info("[vllm-cpp] resolved DFlash draft checkpoint", "reference", ref, "path", abs)
+		return string(out), nil
+	}
+
+	return "", fmt.Errorf(
+		"vllm-cpp: DFlash draft checkpoint %q not found (looked in: %s). "+
+			"The engine does not download drafts - install the draft model into LocalAI first, "+
+			"or set speculative_config.model to an absolute path to a directory containing config.json",
+		ref, strings.Join(candidates, ", "))
 }
 
 func parseInt32(s string, fallback int32) int32 {

backend/go/vllm-cpp/vllmcpp_test.go

@@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) {
 	RunSpecs(t, "vllm-cpp suite")
 }
 
-// The Go POD mirrors must match the C struct layout of vllm.h (ABI v2)
+// The Go POD mirrors must match the C struct layout of vllm.h (ABI v9)
 // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
 // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
 var _ = Describe("C ABI struct mirrors", func() {
@@ -30,10 +30,18 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.MaxNumSeqs)).To(Equal(uintptr(28)))
 		Expect(unsafe.Offsetof(p.ToolParser)).To(Equal(uintptr(32)))
 		Expect(unsafe.Offsetof(p.ReasoningParser)).To(Equal(uintptr(40)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(48)))
+		Expect(unsafe.Offsetof(p.SpeculativeConfig)).To(Equal(uintptr(48)))
+		Expect(unsafe.Offsetof(p.EnablePrefixCaching)).To(Equal(uintptr(56)))
+		Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60)))
+		Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64)))
+		Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72)))
+		Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80)))
+		// 88, not 84: the struct is 8-aligned (it holds pointers), so the
+		// trailing int32 is padded out. Go pads identically.
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88)))
 	})
 
-	It("cSamplingParams matches vllm_sampling_params (ABI v2)", func() {
+	It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() {
 		var p cSamplingParams
 		Expect(unsafe.Offsetof(p.Temperature)).To(Equal(uintptr(0)))
 		Expect(unsafe.Offsetof(p.TopP)).To(Equal(uintptr(4)))
@@ -55,7 +63,9 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.NStructuredChoice)).To(Equal(uintptr(96)))
 		Expect(unsafe.Offsetof(p.StructuredGrammar)).To(Equal(uintptr(104)))
 		Expect(unsafe.Offsetof(p.StructuredJSONObject)).To(Equal(uintptr(112)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120)))
+		Expect(unsafe.Offsetof(p.LogitsProcessor)).To(Equal(uintptr(120)))
+		Expect(unsafe.Offsetof(p.LogitsProcessorUserData)).To(Equal(uintptr(128)))
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136)))
 	})
 
 	It("cCompletion matches vllm_completion", func() {
@@ -68,6 +78,23 @@ var _ = Describe("C ABI struct mirrors", func() {
 	})
 })
 
+// Pin/mirror skew is the failure mode this backend is most exposed to: the Go
+// PODs above are hand-written against one VLLM_ABI_VERSION, and the Makefile
+// pins the vllm.cpp commit that produces it. This spec catches drift without
+// needing model weights - set VLLM_CPP_LIBRARY to a built libvllm and it binds
+// every symbol and compares the library's reported ABI against the mirrors'.
+var _ = Describe("real library ABI handshake", func() {
+	It("binds every symbol and reports the ABI the mirrors were written against", func() {
+		lib := os.Getenv("VLLM_CPP_LIBRARY")
+		if lib == "" {
+			Skip("VLLM_CPP_LIBRARY not set; skipping the real-library handshake")
+		}
+		Expect(registerLib(lib)).To(Succeed())
+		Expect(vllmABIVersion()).To(Equal(int32(abiVersion)))
+		Expect(vllmVersion()).NotTo(BeEmpty())
+	})
+})
+
 var _ = Describe("parseOptions", func() {
 	It("extracts the engine sizing knobs", func() {
 		lo := parseOptions(&pb.ModelOptions{Options: []string{
@@ -83,6 +110,129 @@ var _ = Describe("parseOptions", func() {
 		}})
 		Expect(lo).To(Equal(loadOptions{}))
 	})
+
+	It("carries a speculative_config JSON value through the legacy options list", func() {
+		// strings.Cut splits on the FIRST colon only, so a JSON object value
+		// survives the "key:value" spelling intact.
+		lo := parseOptions(&pb.ModelOptions{Options: []string{
+			`speculative_config:{"method":"mtp","num_speculative_tokens":1}`,
+		}})
+		Expect(lo.speculativeConfig).To(Equal(`{"method":"mtp","num_speculative_tokens":1}`))
+	})
+})
+
+var _ = Describe("engine_args", func() {
+	It("maps every load knob onto the C model params", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
+			"block_size": 64,
+			"num_blocks": 1024,
+			"max_model_len": 16384,
+			"max_num_seqs": 32,
+			"max_num_batched_tokens": 8192,
+			"enable_prefix_caching": true,
+			"scheduling_policy": "lpm",
+			"tool_parser": "qwen3",
+			"reasoning_parser": "deepseek_r1",
+			"tokenizer_config": "/models/tok/tokenizer_config.json"
+		}`})
+		Expect(lo.blockSize).To(Equal(int32(64)))
+		Expect(lo.numBlocks).To(Equal(int32(1024)))
+		Expect(lo.maxModelLen).To(Equal(int32(16384)))
+		Expect(lo.maxNumSeqs).To(Equal(int32(32)))
+		Expect(lo.maxNumBatchedTokens).To(Equal(int32(8192)))
+		Expect(lo.enablePrefixCaching).To(Equal(int32(1)))
+		Expect(lo.schedulingPolicy).To(Equal("lpm"))
+		Expect(lo.toolParser).To(Equal("qwen3"))
+		Expect(lo.reasoningParser).To(Equal("deepseek_r1"))
+		Expect(lo.tokenizerConfigPath).To(Equal("/models/tok/tokenizer_config.json"))
+	})
+
+	It("re-marshals a nested speculative_config object to JSON for the engine", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
+			"speculative_config": {"method": "mtp", "num_speculative_tokens": 1}
+		}`})
+		Expect(lo.speculativeConfig).To(MatchJSON(`{"method":"mtp","num_speculative_tokens":1}`))
+	})
+
+	It("re-marshals a nested kv_transfer_config object (LMCache) to JSON", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
+			"kv_transfer_config": {
+				"kv_connector": "LMCacheConnector",
+				"kv_role": "kv_both",
+				"kv_connector_extra_config": {"host": "127.0.0.1", "port": 65432}
+			}
+		}`})
+		Expect(lo.kvTransferConfig).To(MatchJSON(`{
+			"kv_connector":"LMCacheConnector",
+			"kv_role":"kv_both",
+			"kv_connector_extra_config":{"host":"127.0.0.1","port":65432}
+		}`))
+	})
+
+	It("accepts a pre-encoded JSON string for the object-valued knobs", func() {
+		// A config written by hand (or round-tripped through a flat store) may
+		// carry the object as a string; both spellings reach the engine the same.
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{
+			"speculative_config": "{\"method\":\"ngram\",\"num_speculative_tokens\":4}"
+		}`})
+		Expect(lo.speculativeConfig).To(MatchJSON(`{"method":"ngram","num_speculative_tokens":4}`))
+	})
+
+	It("maps enable_prefix_caching false onto the force-OFF tri-state", func() {
+		// The C ABI tri-state is 0=model default, 1=on, 2=off, so an explicit
+		// `false` must NOT collapse to the 0 that means "let the model decide".
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_prefix_caching": false}`})
+		Expect(lo.enablePrefixCaching).To(Equal(int32(2)))
+	})
+
+	It("leaves the prefix-caching tri-state at the model default when unset", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`})
+		Expect(lo.enablePrefixCaching).To(Equal(int32(0)))
+	})
+
+	It("accepts the radix-attention alias upstream documents for prefix caching", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_radix_attention": true}`})
+		Expect(lo.enablePrefixCaching).To(Equal(int32(1)))
+	})
+
+	It("maps enable_jump_forward onto its own tri-state", func() {
+		// ABI v10. Same tri-state shape as prefix caching, and the same trap:
+		// an explicit false must be force-OFF (2), not the 0 that defers to the
+		// environment.
+		on := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": true}`})
+		Expect(on.enableJumpForward).To(Equal(int32(1)))
+		off := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": false}`})
+		Expect(off.enableJumpForward).To(Equal(int32(2)))
+		unset := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`})
+		Expect(unset.enableJumpForward).To(Equal(int32(0)))
+	})
+
+	It("reads enable_jump_forward from the legacy options list too", func() {
+		lo := parseOptions(&pb.ModelOptions{Options: []string{"enable_jump_forward:true"}})
+		Expect(lo.enableJumpForward).To(Equal(int32(1)))
+	})
+
+	It("lets engine_args override the legacy options list", func() {
+		lo := parseOptions(&pb.ModelOptions{
+			Options:    []string{"max_num_seqs:8", "block_size:16"},
+			EngineArgs: `{"max_num_seqs": 64}`,
+		})
+		Expect(lo.maxNumSeqs).To(Equal(int32(64))) // engine_args wins
+		Expect(lo.blockSize).To(Equal(int32(16)))  // untouched keys survive
+	})
+
+	It("ignores malformed engine_args rather than failing the load", func() {
+		lo := parseOptions(&pb.ModelOptions{
+			Options:    []string{"max_num_seqs:8"},
+			EngineArgs: `{not json`,
+		})
+		Expect(lo.maxNumSeqs).To(Equal(int32(8)))
+	})
+
+	It("ignores unknown keys", func() {
+		lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"gpu_memory_utilization": 0.9}`})
+		Expect(lo).To(Equal(loadOptions{}))
+	})
 })
 
 var _ = Describe("samplingFromPredict", func() {
@@ -135,6 +285,91 @@ var _ = Describe("samplingFromPredict", func() {
 	})
 })
 
+// The engine resolves speculative_config.model against a local directory or
+// ~/.cache/huggingface/hub ONLY - it never downloads. LocalAI keeps models in
+// its own directory, so a bare repo id would miss the HF cache and fail deep in
+// the load with a confusing "draft checkpoint not found". Resolve it here.
+var _ = Describe("resolveDraftModelPath", func() {
+	var modelsDir string
+
+	BeforeEach(func() {
+		modelsDir = GinkgoT().TempDir()
+	})
+
+	// draftDir creates a plausible draft checkpoint under models/.
+	draftDir := func(name string) string {
+		d := filepath.Join(modelsDir, name)
+		Expect(os.MkdirAll(d, 0o750)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(d, "config.json"), []byte("{}"), 0o600)).To(Succeed())
+		return d
+	}
+
+	It("rewrites a repo id to the matching directory in the models dir", func() {
+		want := draftDir("Qwen3.6-27B-DFlash")
+		spec := `{"method":"dflash","model":"z-lab/Qwen3.6-27B-DFlash"}`
+		out, err := resolveDraftModelPath(spec, modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(MatchJSON(`{"method":"dflash","model":"` + want + `"}`))
+	})
+
+	It("rewrites a models-dir-relative path", func() {
+		want := draftDir("drafts__dflash")
+		spec := `{"method":"dflash","model":"drafts__dflash"}`
+		out, err := resolveDraftModelPath(spec, modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(ContainSubstring(want))
+	})
+
+	It("leaves an absolute path that already resolves alone", func() {
+		abs := draftDir("elsewhere")
+		spec := `{"method":"dflash","model":"` + abs + `"}`
+		out, err := resolveDraftModelPath(spec, modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(MatchJSON(spec))
+	})
+
+	It("fails with an actionable error when the draft is nowhere on disk", func() {
+		// Silently passing the repo id through would surface as an HF-cache
+		// miss inside the engine, which reads as "your model is broken".
+		spec := `{"method":"dflash","model":"z-lab/Not-Downloaded"}`
+		_, err := resolveDraftModelPath(spec, modelsDir)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("z-lab/Not-Downloaded"))
+		Expect(err.Error()).To(ContainSubstring(modelsDir))
+	})
+
+	It("requires a model key for dflash", func() {
+		_, err := resolveDraftModelPath(`{"method":"dflash"}`, modelsDir)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("model"))
+	})
+
+	It("leaves mtp and ngram configs untouched", func() {
+		// Neither has a separate draft checkpoint to resolve.
+		for _, spec := range []string{
+			`{"method":"mtp"}`,
+			`{"method":"ngram","num_speculative_tokens":4}`,
+		} {
+			out, err := resolveDraftModelPath(spec, modelsDir)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(MatchJSON(spec))
+		}
+	})
+
+	It("passes a malformed document through for the engine to reject", func() {
+		// The engine owns config validation and produces the better message.
+		out, err := resolveDraftModelPath(`{not json`, modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(Equal(`{not json`))
+	})
+
+	It("is a no-op on an empty config", func() {
+		out, err := resolveDraftModelPath("", modelsDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out).To(BeEmpty())
+	})
+})
+
 var _ = Describe("validModelPath", func() {
 	It("accepts a .gguf file", func() {
 		dir := GinkgoT().TempDir()

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=2ca53bb45e38748d07b310eeb36245a7157ac882
+WHISPER_CPP_VERSION?=64d57d3df5c8dacee098577257edcaa154bf5ef3
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

core/config/vllm_spec.go

@@ -0,0 +1,117 @@
+package config
+
+// Speculative-decoding auto-defaults for the vllm-cpp backend, the safetensors
+// counterpart of the GGUF/llama.cpp hook in mtp.go.
+//
+// The two engines detect and spell the same feature differently. llama.cpp
+// reads `<arch>.nextn_predict_layers` out of the GGUF header and takes
+// `spec_type:draft-mtp` in `options:`; vllm.cpp reads `mtp_num_hidden_layers`
+// out of the checkpoint's config.json and takes vLLM's own
+// `--speculative-config` JSON, which LocalAI carries in `engine_args`. The
+// engine resolves the draft depth and the default k itself, so the config only
+// has to name the method.
+
+import (
+	"encoding/json"
+
+	"github.com/mudler/xlog"
+)
+
+// hfSpecConfig is the subset of a HuggingFace config.json that decides whether
+// speculative decoding can be auto-enabled.
+type hfSpecConfig struct {
+	ModelType string `json:"model_type"`
+	// MtpNumHiddenLayers is the MTP head depth (upstream speculative.py reads
+	// it as n_predict for the qwen3_5 / qwen3_5_moe families).
+	MtpNumHiddenLayers uint32 `json:"mtp_num_hidden_layers"`
+	// DFlashConfig marks a z-lab DFlash DRAFT checkpoint (mask_token_id +
+	// target_layer_ids). Its presence means this repo is a draft, not a
+	// servable target.
+	DFlashConfig json.RawMessage `json:"dflash_config"`
+	// TextConfig is where multimodal checkpoints nest the language-model
+	// config, and therefore the MTP depth.
+	TextConfig *hfSpecConfig `json:"text_config"`
+}
+
+// parseHFSpecConfig decodes the speculative-relevant subset of a config.json.
+// A document that does not parse yields nothing rather than an error: detection
+// is best-effort and must never break an import.
+func parseHFSpecConfig(configJSON []byte) (hfSpecConfig, bool) {
+	if len(configJSON) == 0 {
+		return hfSpecConfig{}, false
+	}
+	var c hfSpecConfig
+	if err := json.Unmarshal(configJSON, &c); err != nil {
+		xlog.Debug("[vllm-spec] config.json did not parse; skipping detection", "error", err)
+		return hfSpecConfig{}, false
+	}
+	return c, true
+}
+
+// IsDFlashDraftConfig reports whether a HuggingFace config.json describes a
+// DFlash DRAFT checkpoint. Unlike MTP - whose head ships inside the target
+// checkpoint's `mtp.*` tensors - a DFlash draft is its own repo that can only
+// run paired with a target it verifies against, so it must never be configured
+// as a standalone model.
+func IsDFlashDraftConfig(configJSON []byte) bool {
+	c, ok := parseHFSpecConfig(configJSON)
+	if !ok {
+		return false
+	}
+	return len(c.DFlashConfig) > 0 ||
+		(c.TextConfig != nil && len(c.TextConfig.DFlashConfig) > 0)
+}
+
+// HasSafetensorsMTPHead reports whether a HuggingFace config.json declares a
+// self-speculating Multi-Token Prediction head, returning its depth. The depth
+// is informational: vllm.cpp resolves n_predict and the default
+// num_speculative_tokens from the checkpoint itself.
+//
+// DFlash drafts are excluded for the same reason `gemma4-assistant` GGUFs are
+// excluded from the llama.cpp hook: they carry head metadata but cannot
+// self-speculate.
+//
+// NOTE this is a safetensors-only signal. vllm.cpp rejects an MTP config over a
+// GGUF source, because the `mtp.*` draft tensors only exist in the safetensors
+// checkpoint - so the GGUF import path must not use this.
+func HasSafetensorsMTPHead(configJSON []byte) (uint32, bool) {
+	c, ok := parseHFSpecConfig(configJSON)
+	if !ok {
+		return 0, false
+	}
+	if IsDFlashDraftConfig(configJSON) {
+		return 0, false
+	}
+	n := c.MtpNumHiddenLayers
+	if n == 0 && c.TextConfig != nil {
+		n = c.TextConfig.MtpNumHiddenLayers
+	}
+	return n, n > 0
+}
+
+// ApplyVLLMSpeculativeDefaults enables MTP speculative decoding in cfg's
+// engine_args when nothing is configured there yet. It is a no-op when the user
+// already set a speculative_config, so an explicit choice (a different method,
+// an explicit k, a DFlash draft) is never clobbered.
+//
+// `layers` is the detected head depth and is only used for the diagnostic log
+// line - the engine derives the real k from the checkpoint.
+func ApplyVLLMSpeculativeDefaults(cfg *ModelConfig, layers uint32) {
+	if cfg == nil {
+		return
+	}
+	if _, set := cfg.EngineArgs["speculative_config"]; set {
+		xlog.Debug("[vllm-spec] MTP head detected but speculative_config already configured; leaving user choice intact",
+			"name", cfg.Name, "mtp_num_hidden_layers", layers)
+		return
+	}
+	if cfg.EngineArgs == nil {
+		cfg.EngineArgs = map[string]any{}
+	}
+	// Only the method: vllm.cpp defaults num_speculative_tokens to the
+	// checkpoint's own n_predict (speculative.py:865-875), which is the right
+	// value far more reliably than anything guessable here.
+	cfg.EngineArgs["speculative_config"] = map[string]any{"method": "mtp"}
+	xlog.Info("[vllm-spec] MTP head detected; enabling mtp speculative decoding",
+		"name", cfg.Name, "mtp_num_hidden_layers", layers)
+}

core/config/vllm_spec_test.go

@@ -0,0 +1,117 @@
+package config_test
+
+import (
+	. "github.com/mudler/LocalAI/core/config"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("vllm-cpp speculative-decoding auto-defaults", func() {
+	Context("HasSafetensorsMTPHead", func() {
+		It("detects a top-level mtp_num_hidden_layers", func() {
+			n, ok := HasSafetensorsMTPHead([]byte(`{
+				"model_type": "qwen3_5_moe",
+				"mtp_num_hidden_layers": 1
+			}`))
+			Expect(ok).To(BeTrue())
+			Expect(n).To(Equal(uint32(1)))
+		})
+
+		It("detects the head nested under text_config", func() {
+			// Multimodal checkpoints nest the language-model config, which is
+			// where the MTP depth lives (mirrors the engine's own resolution
+			// off config.raw text_config).
+			n, ok := HasSafetensorsMTPHead([]byte(`{
+				"model_type": "qwen3_5_moe",
+				"text_config": {"mtp_num_hidden_layers": 2}
+			}`))
+			Expect(ok).To(BeTrue())
+			Expect(n).To(Equal(uint32(2)))
+		})
+
+		It("reports no head when the key is absent", func() {
+			n, ok := HasSafetensorsMTPHead([]byte(`{"model_type": "llama"}`))
+			Expect(ok).To(BeFalse())
+			Expect(n).To(BeZero())
+		})
+
+		It("reports no head for a zero depth", func() {
+			_, ok := HasSafetensorsMTPHead([]byte(`{"mtp_num_hidden_layers": 0}`))
+			Expect(ok).To(BeFalse())
+		})
+
+		It("ignores a DFlash draft checkpoint", func() {
+			// A DFlash draft is a SEPARATE checkpoint that cannot serve alone:
+			// it needs a target to verify against. Same exclusion the GGUF path
+			// makes for gemma4-assistant drafts.
+			_, ok := HasSafetensorsMTPHead([]byte(`{
+				"model_type": "qwen3_dflash",
+				"mtp_num_hidden_layers": 1,
+				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0, 1]}
+			}`))
+			Expect(ok).To(BeFalse())
+		})
+
+		It("reports no head on unparseable JSON", func() {
+			_, ok := HasSafetensorsMTPHead([]byte(`{not json`))
+			Expect(ok).To(BeFalse())
+		})
+
+		It("reports no head on empty input", func() {
+			_, ok := HasSafetensorsMTPHead(nil)
+			Expect(ok).To(BeFalse())
+		})
+	})
+
+	Context("IsDFlashDraftConfig", func() {
+		It("recognises a draft by its dflash_config block", func() {
+			Expect(IsDFlashDraftConfig([]byte(`{
+				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0]}
+			}`))).To(BeTrue())
+		})
+
+		It("does not flag an ordinary checkpoint", func() {
+			Expect(IsDFlashDraftConfig([]byte(`{"model_type": "qwen3_5_moe"}`))).To(BeFalse())
+		})
+	})
+
+	Context("ApplyVLLMSpeculativeDefaults", func() {
+		It("writes the mtp method into engine_args", func() {
+			cfg := &ModelConfig{Name: "qwen"}
+			ApplyVLLMSpeculativeDefaults(cfg, 1)
+			Expect(cfg.EngineArgs).To(HaveKey("speculative_config"))
+			spec, ok := cfg.EngineArgs["speculative_config"].(map[string]any)
+			Expect(ok).To(BeTrue())
+			Expect(spec["method"]).To(Equal("mtp"))
+		})
+
+		It("leaves an existing speculative_config alone", func() {
+			cfg := &ModelConfig{
+				Name: "qwen",
+				LLMConfig: LLMConfig{
+					EngineArgs: map[string]any{
+						"speculative_config": map[string]any{"method": "ngram", "num_speculative_tokens": 4},
+					},
+				},
+			}
+			ApplyVLLMSpeculativeDefaults(cfg, 1)
+			spec := cfg.EngineArgs["speculative_config"].(map[string]any)
+			Expect(spec["method"]).To(Equal("ngram"))
+		})
+
+		It("preserves unrelated engine_args keys", func() {
+			cfg := &ModelConfig{
+				Name:      "qwen",
+				LLMConfig: LLMConfig{EngineArgs: map[string]any{"max_num_seqs": 32}},
+			}
+			ApplyVLLMSpeculativeDefaults(cfg, 1)
+			Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_seqs", 32))
+			Expect(cfg.EngineArgs).To(HaveKey("speculative_config"))
+		})
+
+		It("tolerates a nil config", func() {
+			Expect(func() { ApplyVLLMSpeculativeDefaults(nil, 1) }).ToNot(Panic())
+		})
+	})
+})

core/gallery/importers/llama-cpp.go

@@ -298,7 +298,15 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
 	// imported configs already carry spec_type:draft-mtp before the model is
 	// ever loaded - users see it in the YAML preview rather than discovering
 	// it after the first start.
-	maybeApplyMTPDefaults(&modelConfig, details, &cfg)
+	//
+	// vllm-cpp is excluded on both counts: `spec_type:*` are llama.cpp option
+	// keys it does not read, and vllm.cpp rejects an MTP config over a GGUF
+	// source outright (the `mtp.*` draft tensors exist only in the safetensors
+	// checkpoint). Its MTP auto-config runs in the vllm importer instead, over
+	// the safetensors config.json.
+	if backend != "vllm-cpp" {
+		maybeApplyMTPDefaults(&modelConfig, details, &cfg)
+	}
 
 	data, err := yaml.Marshal(modelConfig)
 	if err != nil {

core/gallery/importers/vllm.go

@@ -1,13 +1,21 @@
 package importers
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/gallery"
 	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/pkg/downloader"
+	"github.com/mudler/LocalAI/pkg/httpclient"
+	"github.com/mudler/xlog"
 	"go.yaml.in/yaml/v2"
 )
 
@@ -107,6 +115,12 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
 		// vllm python backend, so use_tokenizer_template carries over), but
 		// tool/reasoning parsing is the engine's own autoparser pipeline -
 		// the vllm-python tool_parser/reasoning_parser options don't apply.
+		//
+		// Auto-detect a Multi-Token Prediction head, the safetensors analogue
+		// of the llama-cpp importer's GGUF hook, so a freshly imported
+		// Qwen3.5 / Qwen3.6 config already carries speculative decoding in its
+		// engine_args instead of leaving the throughput on the table.
+		maybeApplyVLLMSpeculativeDefaults(&modelConfig, details)
 	} else {
 		// Auto-detect tool_parser and reasoning_parser for known model families.
 		// Surfacing them in the generated YAML lets users see and edit the choices.
@@ -132,3 +146,89 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
 		ConfigFile:  string(data),
 	}, nil
 }
+
+// maxSpecConfigProbeBytes caps the config.json body we read. Real ones are a
+// few KB; the cap keeps a hostile or mislabelled URL from streaming into the
+// importer.
+const maxSpecConfigProbeBytes = 1 << 20 // 1 MiB
+
+// specConfigProbeTimeout bounds the config.json fetch. Detection is an
+// optimisation, so it must never hold an import open for long.
+const specConfigProbeTimeout = 30 * time.Second
+
+// specConfigFetcher is the seam the config.json probe goes through, so tests can
+// drive the whole import path without a network round trip.
+var specConfigFetcher = fetchProbeBody
+
+// maybeApplyVLLMSpeculativeDefaults fetches the repository's config.json and,
+// when it declares a Multi-Token Prediction head, enables MTP speculative
+// decoding in the emitted engine_args. This is the safetensors counterpart of
+// the llama-cpp importer's GGUF header probe.
+//
+// Every failure is non-fatal and logged at debug: a network blip, a private
+// repo, or a config.json this doesn't understand must leave the import working
+// exactly as it did before, just without the speculative default.
+func maybeApplyVLLMSpeculativeDefaults(modelConfig *config.ModelConfig, details Details) {
+	probeURL := vllmSpecProbeURL(details)
+	if probeURL == "" {
+		return
+	}
+
+	body, err := specConfigFetcher(probeURL)
+	if err != nil {
+		xlog.Debug("[vllm-spec-importer] could not read config.json for MTP detection", "uri", probeURL, "error", err)
+		return
+	}
+
+	applySpecFromConfigJSON(modelConfig, body, details.URI)
+}
+
+// applySpecFromConfigJSON is the decision half of the probe, split out so it can
+// be exercised without a network round trip.
+func applySpecFromConfigJSON(modelConfig *config.ModelConfig, body []byte, uri string) {
+	if config.IsDFlashDraftConfig(body) {
+		// A DFlash draft cannot serve on its own - it only proposes tokens for
+		// a target model to verify. Say so rather than emitting a config that
+		// would fail at load.
+		xlog.Warn("[vllm-spec-importer] this repository is a DFlash DRAFT checkpoint, not a servable model; "+
+			"import the TARGET model and point engine_args.speculative_config at this repo "+
+			`({"method":"dflash","model":"<this repo>"})`, "uri", uri)
+		return
+	}
+
+	n, ok := config.HasSafetensorsMTPHead(body)
+	if !ok {
+		return
+	}
+	config.ApplyVLLMSpeculativeDefaults(modelConfig, n)
+}
+
+// vllmSpecProbeURL returns the HTTP(S) URL of the repository's config.json, or
+// "" when the import isn't backed by a HuggingFace repo we can fetch from (a
+// local directory import, an OCI artifact, ...).
+func vllmSpecProbeURL(details Details) string {
+	if details.HuggingFace == nil || details.HuggingFace.ModelID == "" {
+		return ""
+	}
+	return resolveHTTPProbe(downloader.HuggingFacePrefix + details.HuggingFace.ModelID + "/config.json")
+}
+
+// fetchProbeBody GETs a small remote JSON document under a short timeout.
+func fetchProbeBody(url string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), specConfigProbeTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := httpclient.NewWithTimeout(specConfigProbeTimeout).Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status %d", resp.StatusCode)
+	}
+	return io.ReadAll(io.LimitReader(resp.Body, maxSpecConfigProbeBytes))
+}

core/gallery/importers/vllm_spec_internal_test.go

@@ -0,0 +1,118 @@
+package importers
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/mudler/LocalAI/core/config"
+	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("vllm-cpp speculative auto-config (importer)", func() {
+	Context("applySpecFromConfigJSON", func() {
+		It("enables mtp when the checkpoint declares an MTP head", func() {
+			cfg := &config.ModelConfig{Name: "qwen3.5"}
+			applySpecFromConfigJSON(cfg, []byte(`{
+				"model_type": "qwen3_5_moe",
+				"mtp_num_hidden_layers": 1
+			}`), "huggingface://Qwen/Qwen3.5-A3B")
+			Expect(cfg.EngineArgs).To(HaveKeyWithValue("speculative_config",
+				map[string]any{"method": "mtp"}))
+		})
+
+		It("leaves a plain checkpoint untouched", func() {
+			cfg := &config.ModelConfig{Name: "llama"}
+			applySpecFromConfigJSON(cfg, []byte(`{"model_type": "llama"}`), "huggingface://meta/llama")
+			Expect(cfg.EngineArgs).To(BeEmpty())
+		})
+
+		It("refuses to configure a DFlash draft as a servable model", func() {
+			// The draft only proposes tokens; configuring it standalone would
+			// produce a model that cannot load.
+			cfg := &config.ModelConfig{Name: "dflash-draft"}
+			applySpecFromConfigJSON(cfg, []byte(`{
+				"model_type": "qwen3_dflash",
+				"dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0, 1]}
+			}`), "huggingface://z-lab/Qwen3.6-27B-DFlash")
+			Expect(cfg.EngineArgs).To(BeEmpty())
+		})
+
+		It("survives a config.json it cannot parse", func() {
+			cfg := &config.ModelConfig{Name: "weird"}
+			Expect(func() {
+				applySpecFromConfigJSON(cfg, []byte(`<html>404</html>`), "huggingface://a/b")
+			}).ToNot(Panic())
+			Expect(cfg.EngineArgs).To(BeEmpty())
+		})
+	})
+
+	Context("Import over a repository with an MTP head", func() {
+		var restore func()
+
+		BeforeEach(func() {
+			original := specConfigFetcher
+			restore = func() { specConfigFetcher = original }
+		})
+		AfterEach(func() { restore() })
+
+		importWith := func(backend, configJSON string) string {
+			specConfigFetcher = func(string) ([]byte, error) {
+				return []byte(configJSON), nil
+			}
+			importer := &VLLMImporter{}
+			out, err := importer.Import(Details{
+				URI:         "huggingface://Qwen/Qwen3.5-A3B",
+				Preferences: json.RawMessage(`{"backend": "` + backend + `"}`),
+				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
+			})
+			Expect(err).ToNot(HaveOccurred())
+			return out.ConfigFile
+		}
+
+		It("emits engine_args.speculative_config for vllm-cpp", func() {
+			yaml := importWith("vllm-cpp", `{"model_type":"qwen3_5_moe","mtp_num_hidden_layers":1}`)
+			Expect(yaml).To(ContainSubstring("engine_args:"))
+			Expect(yaml).To(ContainSubstring("speculative_config:"))
+			Expect(yaml).To(ContainSubstring("method: mtp"))
+		})
+
+		It("emits nothing speculative for the python vllm backend", func() {
+			// The python backend has its own speculative surface and its own
+			// version-dependent MTP support; this hook is vllm-cpp only.
+			yaml := importWith("vllm", `{"model_type":"qwen3_5_moe","mtp_num_hidden_layers":1}`)
+			Expect(yaml).NotTo(ContainSubstring("speculative_config"))
+		})
+
+		It("emits nothing speculative when the probe fails", func() {
+			specConfigFetcher = func(string) ([]byte, error) {
+				return nil, errors.New("network down")
+			}
+			importer := &VLLMImporter{}
+			out, err := importer.Import(Details{
+				URI:         "huggingface://Qwen/Qwen3.5-A3B",
+				Preferences: json.RawMessage(`{"backend": "vllm-cpp"}`),
+				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out.ConfigFile).NotTo(ContainSubstring("speculative_config"))
+		})
+	})
+
+	Context("vllmSpecProbeURL", func() {
+		It("resolves the repository's config.json to an HTTPS URL", func() {
+			url := vllmSpecProbeURL(Details{
+				URI:         "huggingface://Qwen/Qwen3.5-A3B",
+				HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"},
+			})
+			Expect(url).To(ContainSubstring("Qwen/Qwen3.5-A3B"))
+			Expect(url).To(HaveSuffix("config.json"))
+			Expect(url).To(HavePrefix("https://"))
+		})
+
+		It("skips the probe when there is no HuggingFace repo behind the import", func() {
+			Expect(vllmSpecProbeURL(Details{URI: "/models/local-dir"})).To(BeEmpty())
+		})
+	})
+})

core/http/react-ui/package-lock.json

@@ -21,9 +21,10 @@
         "@fortawesome/fontawesome-free": "^6.7.2",
         "@lezer/highlight": "^1.2.1",
         "@modelcontextprotocol/ext-apps": "^1.2.2",
-        "@modelcontextprotocol/sdk": "^1.25.1",
+        "@modelcontextprotocol/sdk": "^1.30.0",
         "dompurify": "^3.4.12",
         "highlight.js": "^11.11.1",
+        "hono": "4.12.25",
         "i18next": "^26.0.8",
         "i18next-browser-languagedetector": "^8.2.1",
         "i18next-http-backend": "^3.0.6",
@@ -944,11 +945,12 @@
       }
     },
     "node_modules/@modelcontextprotocol/sdk": {
-      "version": "1.27.1",
-      "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.27.1.tgz",
-      "integrity": "sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA==",
+      "version": "1.30.0",
+      "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz",
+      "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==",
+      "license": "MIT",
       "dependencies": {
-        "@hono/node-server": "^1.19.9",
+        "@hono/node-server": "^1.19.9 || ^2.0.5",
         "ajv": "^8.17.1",
         "ajv-formats": "^3.0.1",
         "content-type": "^1.0.5",
@@ -1718,10 +1720,11 @@
       "dev": true
     },
     "node_modules/brace-expansion": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
-      "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
+      "version": "1.1.18",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz",
+      "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -3432,9 +3435,9 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.12.31",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.31.tgz",
-      "integrity": "sha512-zJIHFrl6bq3RDd2YusFNCDlM8qUprxKswyi/OPzPyzKDdyBXDqWx8bZlZ7R+saTdSTatUmb3O7K4SspGPaEOQg==",
+      "version": "4.12.25",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.25.tgz",
+      "integrity": "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==",
       "license": "MIT",
       "engines": {
         "node": ">=16.9.0"
@@ -4383,16 +4386,16 @@
       }
     },
     "node_modules/istanbul-lib-processinfo/node_modules/brace-expansion": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
       },
       "engines": {
-        "node": "18 || 20 || >=22"
+        "node": "20 || >=22"
       }
     },
     "node_modules/istanbul-lib-processinfo/node_modules/glob": {
@@ -5278,16 +5281,16 @@
       }
     },
     "node_modules/nyc/node_modules/brace-expansion": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
       },
       "engines": {
-        "node": "18 || 20 || >=22"
+        "node": "20 || >=22"
       }
     },
     "node_modules/nyc/node_modules/convert-source-map": {
@@ -5974,10 +5977,11 @@
       }
     },
     "node_modules/quick-temp/node_modules/brace-expansion": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz",
-      "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==",
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
+      "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0"
       }
@@ -6569,16 +6573,16 @@
       }
     },
     "node_modules/spawn-wrap/node_modules/brace-expansion": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
       },
       "engines": {
-        "node": "18 || 20 || >=22"
+        "node": "20 || >=22"
       }
     },
     "node_modules/spawn-wrap/node_modules/foreground-child": {
@@ -6902,16 +6906,16 @@
       }
     },
     "node_modules/test-exclude/node_modules/brace-expansion": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
       },
       "engines": {
-        "node": "18 || 20 || >=22"
+        "node": "20 || >=22"
       }
     },
     "node_modules/test-exclude/node_modules/glob": {

core/http/react-ui/package.json

@@ -35,7 +35,7 @@
     "@fortawesome/fontawesome-free": "^6.7.2",
     "@lezer/highlight": "^1.2.1",
     "@modelcontextprotocol/ext-apps": "^1.2.2",
-    "@modelcontextprotocol/sdk": "^1.25.1",
+    "@modelcontextprotocol/sdk": "^1.30.0",
     "dompurify": "^3.4.12",
     "highlight.js": "^11.11.1",
     "hono": "4.12.25",

docs/content/blog/2024-04-24-reranker-api.md

@@ -1,10 +0,0 @@
-+++
-title = "Reranker API"
-date = 2024-04-24
-description = "A new reranker backend implementing the Jina rerankers API."
-url = "/blog/reranker-api/"
-+++
-
-A new reranker backend lands, implementing the Jina rerankers API, in [PR #2121](https://github.com/mudler/LocalAI/pull/2121).
-
-See [Reranker]({{% relref "features/reranker" %}}).

docs/content/blog/2024-05-14-distributed-and-p2p-inferencing.md

@@ -1,13 +0,0 @@
-+++
-title = "Distributed and decentralized P2P inferencing"
-date = 2024-05-14
-description = "Distributed llama.cpp inferencing, followed by fully decentralized peer-to-peer inference."
-url = "/blog/distributed-and-p2p-inferencing/"
-+++
-
-Two changes that set up everything LocalAI later built on top of:
-
-- [Distributed llama.cpp inferencing](https://github.com/mudler/LocalAI/pull/2324), splitting a model across machines.
-- [Totally decentralized, private, distributed peer-to-peer inference](https://github.com/mudler/LocalAI/pull/2343).
-
-See [Distributed inferencing]({{% relref "features/distributed_inferencing" %}}).

docs/content/blog/2024-08-02-p2p-federation-and-swarms.md

@@ -1,15 +0,0 @@
-+++
-title = "P2P dashboard, federated mode and AI swarms"
-date = 2024-08-02
-description = "A P2P dashboard, federation, AI swarms, global community pools, FLUX-1 support and the P2P Explorer."
-url = "/blog/p2p-federation-and-swarms/"
-+++
-
-The peer-to-peer work matured over July and August:
-
-- [A P2P dashboard, federated mode and AI swarms](https://github.com/mudler/LocalAI/pull/2723).
-- [Global community pools](https://github.com/mudler/LocalAI/issues/3113), for sharing federated instances and workers.
-- FLUX-1 support.
-- The [P2P Explorer](https://explorer.localai.io).
-
-See [Distributed inferencing]({{% relref "features/distributed_inferencing" %}}).

docs/content/blog/2024-10-01-examples-moved-out.md

@@ -1,8 +0,0 @@
-+++
-title = "Examples move to LocalAI-examples"
-date = 2024-10-01
-description = "The examples directory leaves the main repository and gets its own home."
-url = "/blog/examples-moved-out/"
-+++
-
-The examples have moved out of the main repository into [LocalAI-examples](https://github.com/mudler/LocalAI-examples), where they can be versioned and maintained independently of the runtime.

docs/content/blog/2024-11-20-vad-and-bark-cpp.md

@@ -1,9 +0,0 @@
-+++
-title = "Voice Activity Detection and bark.cpp"
-date = 2024-11-20
-description = "Silero-based Voice Activity Detection, plus a bark.cpp backend for audio generation."
-url = "/blog/vad-and-bark-cpp/"
-+++
-
-- [Voice Activity Detection](https://github.com/mudler/LocalAI/pull/4204), via a Silero VAD backend. See [Voice activity detection]({{% relref "features/voice-activity-detection" %}}).
-- [A bark.cpp backend](https://github.com/mudler/LocalAI/pull/4287) for audio generation.

docs/content/blog/2024-12-03-stablediffusion-cpp-backend.md

@@ -1,10 +0,0 @@
-+++
-title = "stablediffusion.cpp backend (ggml)"
-date = 2024-12-03
-description = "A ggml-based stablediffusion.cpp backend for image generation."
-url = "/blog/stablediffusion-cpp-backend/"
-+++
-
-A ggml-based `stablediffusion.cpp` backend lands for image generation, in [PR #4289](https://github.com/mudler/LocalAI/pull/4289).
-
-See [Image generation]({{% relref "features/image-generation" %}}).

docs/content/blog/2025-07-24-modular-backend-architecture.md

@@ -1,12 +0,0 @@
-+++
-title = "Backends move outside the main binary"
-date = 2025-07-24
-description = "All backends migrate out of the main binary, leaving a lightweight modular core that pulls engines on demand."
-url = "/blog/modular-backend-architecture/"
-+++
-
-All backends have been migrated outside the main binary. The core stays small, and each backend is an isolated service installed on demand.
-
-This is the architecture LocalAI still runs on: install, update or remove engines independently, and mix CPU, NVIDIA, AMD, Intel, Apple Silicon, Vulkan and Jetson in one deployment.
-
-See [Backends]({{% relref "features/backends" %}}) and the [v3.2.0 release notes](https://github.com/mudler/LocalAI/releases/tag/v3.2.0).

docs/content/blog/2025-08-12-apple-silicon-backends.md

@@ -1,10 +0,0 @@
-+++
-title = "MLX, MLX-VLM, Diffusers and llama.cpp on Apple Silicon"
-date = 2025-08-12
-description = "Apple Silicon gains first-class backend coverage."
-url = "/blog/apple-silicon-backends/"
-+++
-
-MLX, MLX-VLM, Diffusers and llama.cpp are now supported on Apple Silicon, giving Mac users the same backend choice available elsewhere.
-
-Released as part of [v3.4.0](https://github.com/mudler/LocalAI/releases/tag/v3.4.0).

docs/content/blog/2025-09-03-launcher-and-extended-backends.md

@@ -1,13 +0,0 @@
-+++
-title = "New launcher, extended backend support, MLX-Audio and WAN 2.2"
-date = 2025-09-03
-description = "A desktop launcher for macOS and Linux, wider backend coverage for Mac and Nvidia L4T, MLX-Audio and WAN 2.2."
-url = "/blog/launcher-and-extended-backends/"
-+++
-
-- A new [launcher app](https://github.com/mudler/LocalAI/pull/6127) for macOS and Linux, so LocalAI can be started and managed without the terminal.
-- Extended backend support for Mac and Nvidia L4T.
-- MLX-Audio.
-- WAN 2.2.
-
-Released as part of [v3.5.0](https://github.com/mudler/LocalAI/releases/tag/v3.5.0).

docs/content/blog/2025-10-05-mcp-support.md

@@ -1,10 +0,0 @@
-+++
-title = "Model Context Protocol (MCP) support"
-date = 2025-10-05
-description = "Agentic capabilities through MCP, with a new chat/completion endpoint that can call MCP tools."
-url = "/blog/mcp-support/"
-+++
-
-LocalAI gains [Model Context Protocol](https://modelcontextprotocol.io) support for agentic capabilities, through [a new chat/completion endpoint](https://github.com/mudler/LocalAI/pull/6381) that can reach MCP tools, plus [a UI toggle to enable it](https://github.com/mudler/LocalAI/pull/6400).
-
-See [MCP]({{% relref "features/mcp" %}}).

docs/content/blog/2025-11-24-import-models-via-url-and-chat-history.md

@@ -1,11 +0,0 @@
-+++
-title = "Import models via URL, multiple chats and history"
-date = 2025-11-24
-description = "Point LocalAI at a model URL to import it, and keep several chat threads with their history in the UI."
-url = "/blog/import-models-via-url-and-chat-history/"
-+++
-
-Two usability changes:
-
-- [Import models via URL](https://github.com/mudler/LocalAI/pull/7245). Paste a model URL and LocalAI handles the download and configuration.
-- [Multiple chats and history](https://github.com/mudler/LocalAI/pull/7325) in the UI, so conversations persist and can run in parallel.

docs/content/blog/2025-12-16-memory-reclaimer-and-multi-gpu-fitting.md

@@ -1,12 +0,0 @@
-+++
-title = "Dynamic memory reclaimer, multi-GPU fitting and Vibevoice"
-date = 2025-12-16
-description = "Reclaim GPU memory from idle models, fit llama.cpp models across multiple GPUs automatically, and generate long-form speech with Vibevoice."
-url = "/blog/memory-reclaimer-and-multi-gpu-fitting/"
-+++
-
-Three additions this month:
-
-- [A dynamic memory resource reclaimer](https://github.com/mudler/LocalAI/pull/7583), which frees GPU memory held by idle models.
-- [Automatic multi-GPU model fitting for llama.cpp](https://github.com/mudler/LocalAI/pull/7584), so a model too large for one device is split across several without hand-tuning.
-- [The Vibevoice backend](https://github.com/mudler/LocalAI/pull/7494) for long-form speech.

docs/content/blog/2026-01-18-localai-3-10-0.md

@@ -1,17 +0,0 @@
-+++
-title = "LocalAI 3.10.0"
-date = 2026-01-18
-description = "Anthropic API support, the Open Responses API, video and image generation with LTX-2, unified GPU backends, tool streaming, Moonshine and Pocket-TTS."
-url = "/blog/localai-3-10-0/"
-+++
-
-LocalAI 3.10.0 is out.
-
-- Anthropic API support.
-- The Open Responses API.
-- Video and image generation with LTX-2.
-- Unified GPU backends.
-- Tool streaming.
-- Moonshine and Pocket-TTS.
-
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v3.10.0).

docs/content/blog/2026-02-05-realtime-api-and-ace-step.md

@@ -1,11 +0,0 @@
-+++
-title = "Realtime API and ACE-Step 1.5"
-date = 2026-02-05
-description = "Audio-to-audio with tool calling through the Realtime API, plus ACE-Step 1.5 music generation."
-url = "/blog/realtime-api-and-ace-step/"
-+++
-
-Two additions this month:
-
-- [The Realtime API for audio-to-audio with tool calling](https://github.com/mudler/LocalAI/pull/6245). See [Realtime API]({{% relref "features/openai-realtime" %}}).
-- [ACE-Step 1.5 support](https://github.com/mudler/LocalAI/pull/8396) for music generation.

docs/content/blog/2026-03-14-localai-4-0-0.md

@@ -1,16 +0,0 @@
-+++
-title = "LocalAI 4.0.0: native agentic orchestration"
-date = 2026-03-14
-description = "The Agenthub community hub, a full React UI rewrite with Canvas mode, MCP Apps with tool streaming, WebRTC realtime audio, and MLX-distributed."
-url = "/blog/localai-4-0-0/"
-+++
-
-LocalAI 4.0.0 brings agentic orchestration into the core.
-
-- Native agentic orchestration, with the new [Agenthub](https://agenthub.localai.io) community hub.
-- A full React UI rewrite, including Canvas mode.
-- [MCP Apps and client-side MCP](https://github.com/mudler/LocalAI/pull/8947) with tool streaming.
-- [WebRTC realtime audio](https://github.com/mudler/LocalAI/pull/8790).
-- [MLX-distributed](https://github.com/mudler/LocalAI/pull/8801).
-
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.0.0).

docs/content/blog/2026-04-02-localai-4-1-0.md

@@ -1,17 +0,0 @@
-+++
-title = "LocalAI 4.1.0: LocalAI becomes a control tower"
-date = 2026-04-02
-description = "Distributed cluster mode with VRAM-aware routing and autoscaling, a multi-user platform with OIDC, per-user quotas, in-UI fine-tuning, and a visual pipeline editor."
-url = "/blog/localai-4-1-0/"
-+++
-
-LocalAI 4.1.0 turns LocalAI into a control tower rather than a single inference server.
-
-- Distributed cluster mode, with VRAM-aware smart routing and autoscaling.
-- A multi-user platform with OIDC and API keys.
-- Per-user quotas with predictive analytics.
-- In-UI fine-tuning with TRL, including automatic export to GGUF.
-- An on-the-fly quantization backend.
-- A visual pipeline editor.
-
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.1.0).

docs/content/blog/2026-04-22-face-recognition-backend.md

@@ -1,20 +0,0 @@
-+++
-title = "Face recognition backend"
-date = 2026-04-22
-description = "insightface-powered 1:1 verification, 1:N identification, face embedding, detection and demographic analysis."
-url = "/blog/face-recognition-backend/"
-+++
-
-A new face recognition backend, powered by `insightface`, covering:
-
-- 1:1 verification
-- 1:N identification
-- Face embedding
-- Face detection
-- Demographic analysis
-
-It ships with two model options: the non-commercial `buffalo_l`, and an Apache 2.0 alternative from the OpenCV Zoo.
-
-See [Face recognition]({{% relref "features/face-recognition" %}}). Shipped in [PR #9480](https://github.com/mudler/LocalAI/pull/9480).
-
-The engine was later rewritten from scratch in C++/ggml: see [Native biometric backends]({{% relref "blog/2026-06-28-native-biometric-backends" %}}).

docs/content/blog/2026-05-04-audio-transform.md

@@ -1,19 +0,0 @@
-+++
-title = "Audio Transform"
-date = 2026-05-04
-description = "A generic audio-in / audio-out endpoint with an optional reference signal. First implementation: LocalVQE, a joint AEC, noise suppression and dereverberation engine."
-url = "/blog/audio-transform/"
-+++
-
-Audio Transform is a generic audio-in / audio-out endpoint, with an optional reference signal for tasks that need one.
-
-The first implementation is [LocalVQE](https://github.com/localai-org/LocalVQE), a C++ backend doing joint acoustic echo cancellation, noise suppression and dereverberation in a DeepVQE-style model.
-
-Both call styles are supported:
-
-- Batch, via `POST /audio/transformations`.
-- Bidirectional streaming, via the `/audio/transformations/stream` WebSocket.
-
-Studio gains a "Transform" tab with synchronized waveform players for the input, reference and output signals.
-
-See [Audio transform]({{% relref "features/audio-transform" %}}). Shipped in [PR #9640](https://github.com/mudler/LocalAI/pull/9640).

docs/content/blog/2026-05-05-speaker-diarization.md

@@ -1,17 +0,0 @@
-+++
-title = "Speaker diarization"
-date = 2026-05-05
-description = "A /v1/audio/diarization endpoint returning who spoke when, backed by sherpa-onnx and vibevoice-cpp."
-url = "/blog/speaker-diarization/"
-+++
-
-`POST /v1/audio/diarization` is a new endpoint that returns "who spoke when" as a list of segments.
-
-Two backends serve it:
-
-- `sherpa-onnx` for pure diarization, combining pyannote-3.0, speaker embeddings and clustering.
-- `vibevoice-cpp` for diarization bundled with long-form ASR.
-
-Responses are available as `json`, `verbose_json` or `rttm`.
-
-See [Audio diarization]({{% relref "features/audio-diarization" %}}). Shipped in [PR #9654](https://github.com/mudler/LocalAI/pull/9654).

docs/content/blog/2026-05-24-localai-4-3-0.md

@@ -1,15 +0,0 @@
-+++
-title = "LocalAI 4.3.0"
-date = 2026-05-24
-description = "llama.cpp prompt cache on by default, keyless cosign signing of backend images, per-key and per-user usage attribution, and Distributed v3."
-url = "/blog/localai-4-3-0/"
-+++
-
-LocalAI 4.3.0 is out.
-
-- [Prompt cache on by default for llama.cpp](https://github.com/mudler/LocalAI/pull/9925). Repeated system prompts collapse from minutes to seconds.
-- [Keyless cosign signing of backend OCI images](https://github.com/mudler/LocalAI/pull/9823).
-- [Per-API-key and per-user usage attribution](https://github.com/mudler/LocalAI/pull/9920).
-- Distributed v3, with [per-request replica routing](https://github.com/mudler/LocalAI/pull/9968).
-
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.3.0).

docs/content/blog/2026-06-11-realtime-voice-assistant-demo.md

@@ -1,12 +0,0 @@
-+++
-title = "Realtime voice assistant demo and pipeline streaming"
-date = 2026-06-11
-description = "A tiny Go client for the Realtime API with a full talk-back loop and tool calling, plus streaming of the realtime pipeline stages."
-url = "/blog/realtime-voice-assistant-demo/"
-+++
-
-The new [realtime voice assistant demo](https://github.com/localai-org/localai-realtime-demo) is a small Go client for the Realtime API with a complete talk-back voice loop and tool calling. It is intended as a reference you can read end to end.
-
-On the server side, two supporting changes landed: [streaming of the realtime LLM, TTS and transcription pipeline stages](https://github.com/mudler/LocalAI/pull/10176), and [configurable WebRTC ICE candidates](https://github.com/mudler/LocalAI/pull/10231).
-
-See [Realtime API]({{% relref "features/openai-realtime" %}}).

docs/content/blog/2026-06-12-distributed-mode-hardening.md

@@ -1,16 +0,0 @@
-+++
-title = "Distributed mode hardening"
-date = 2026-06-12
-description = "Prefix-cache-aware routing, a production-ready request router, ds4 layer-split inference, NATS JWT auth with TLS/mTLS, and resumable uploads."
-url = "/blog/distributed-mode-hardening/"
-+++
-
-Distributed mode picked up a round of production hardening:
-
-- [Prefix-cache-aware routing](https://github.com/mudler/LocalAI/pull/10071), so requests sharing a prompt prefix land on the replica that already holds it.
-- [A production-ready request router with auto-sized embedding and rerank batches](https://github.com/mudler/LocalAI/pull/10104).
-- [ds4 layer-split distributed inference](https://github.com/mudler/LocalAI/pull/10098).
-- [NATS JWT auth plus TLS/mTLS](https://github.com/mudler/LocalAI/pull/10159).
-- [Resumable file uploads](https://github.com/mudler/LocalAI/pull/10109).
-
-See [Distributed inferencing]({{% relref "features/distributed_inferencing" %}}).

docs/content/blog/2026-06-12-new-backends-and-models.md

@@ -1,15 +0,0 @@
-+++
-title = "New backends and models: locate-anything.cpp, Ideogram4, Gemma 4"
-date = 2026-06-12
-description = "Open-vocabulary object detection via ggml, Ideogram4 image generation, llama.cpp video input, and the Gemma 4 QAT family with MTP pairs."
-url = "/blog/new-backends-and-models-june-2026/"
-+++
-
-A batch of new capability this month:
-
-- [locate-anything.cpp](https://github.com/mudler/LocalAI/pull/10264) for open-vocabulary object detection via ggml.
-- [Ideogram4 image generation](https://github.com/mudler/LocalAI/pull/10201) in `stablediffusion-ggml`.
-- [llama.cpp video input](https://github.com/mudler/LocalAI/pull/10216).
-- [The Gemma 4 QAT family with MTP speculative-decoding pairs](https://github.com/mudler/LocalAI/pull/10215).
-
-Plus two usability additions: an [interactive CLI chat mode](https://github.com/mudler/LocalAI/pull/10226) and [RAG source citations in agent responses](https://github.com/mudler/LocalAI/pull/10228).

docs/content/blog/2026-06-13-speech-push-parakeet-crispasr-piper.md

@@ -1,17 +0,0 @@
-+++
-title = "A big speech push: parakeet.cpp, CrispASR and 60 Piper voices"
-date = 2026-06-13
-description = "Segment timestamps, multilingual streaming, dynamic batching and CUDA graphs for parakeet.cpp, plus a new ASR/TTS backend and a large Piper voice drop."
-url = "/blog/speech-push-parakeet-crispasr-piper/"
-+++
-
-A concentrated round of speech work landed this month.
-
-[parakeet.cpp](https://github.com/mudler/parakeet.cpp), our ASR engine, gained:
-
-- [NeMo-faithful segment timestamps](https://github.com/mudler/LocalAI/pull/10207)
-- [a multilingual streaming Nemotron-3.5 model](https://github.com/mudler/LocalAI/pull/10199)
-- [dynamic batching for concurrent transcription](https://github.com/mudler/LocalAI/pull/10112)
-- [CUDA graphs](https://github.com/mudler/LocalAI/pull/10273)
-
-Alongside it, the new [CrispASR backend](https://github.com/mudler/LocalAI/pull/10099) adds multi-architecture ASR and TTS, and [60 Piper TTS voices across 42 languages](https://github.com/mudler/LocalAI/pull/10296) land in the gallery, together with [per-request TTS instructions and parameters](https://github.com/mudler/LocalAI/pull/10172).

docs/content/blog/2026-06-18-pii-analyze-redact-api.md

@@ -1,15 +0,0 @@
-+++
-title = "PII analyze and redact API"
-date = 2026-06-18
-description = "The PII detection pipeline becomes a standalone service, callable without routing a chat request through the middleware."
-url = "/blog/pii-analyze-redact-api/"
-+++
-
-The PII detection pipeline (NER plus restricted-regex pattern tiers) is now reachable directly, without routing a chat request through the middleware:
-
-- `POST /api/pii/analyze` returns the detected entity spans.
-- `POST /api/pii/redact` returns the sanitised text, or `400 pii_blocked`.
-
-Events also gain an `origin` field (`middleware`, `proxy`, `pii_analyze`, `pii_redact`), so `/api/pii/events` can be filtered by which surface produced them.
-
-See [Middleware]({{% relref "operations/middleware" %}}#analyze--redact-api). Shipped in [PR #10360](https://github.com/mudler/LocalAI/pull/10360).

docs/content/blog/2026-06-22-sound-classification.md

@@ -1,12 +0,0 @@
-+++
-title = "Sound classification with ced.cpp"
-date = 2026-06-22
-description = "A new /v1/audio/classification endpoint for audio tagging, returning scored AudioSet labels."
-url = "/blog/sound-classification/"
-+++
-
-`POST /v1/audio/classification` is a new endpoint for audio tagging and sound-event classification. It returns scored [AudioSet](https://research.google.com/audioset/) labels: baby cry, glass breaking, alarms, and several hundred others.
-
-It is backed by [ced.cpp](https://github.com/localai-org/ced.cpp), a 527-class AudioSet tagger ported to ggml by the LocalAI team.
-
-See [Audio classification]({{% relref "features/audio-classification" %}}). Shipped in [PR #10425](https://github.com/mudler/LocalAI/pull/10425).

docs/content/blog/2026-06-28-native-biometric-backends.md

@@ -1,15 +0,0 @@
-+++
-title = "Native biometric backends: voice-detect.cpp and face-detect.cpp"
-date = 2026-06-28
-description = "Two from-scratch C++/ggml engines replace the heavier Python insightface and speaker-recognition backends."
-url = "/blog/native-biometric-backends/"
-+++
-
-Two new biometric engines built by the LocalAI team, both from-scratch C++/ggml implementations with no Python and no onnxruntime at inference time:
-
-- [voice-detect.cpp](https://github.com/localai-org/voice-detect.cpp) for speaker recognition and voice analysis: ECAPA-TDNN, WeSpeaker, ERes2Net, CAM++, and wav2vec2 age/gender/emotion.
-- [face-detect.cpp](https://github.com/mudler/face-detect.cpp) for face detection, recognition, demographics and anti-spoofing: SCRFD/ArcFace and YuNet/SFace.
-
-Both ship self-contained GGUF weights, hold bit-exact parity with the reference implementations, and reach cuDNN parity on GPU. They replace the heavier Python `insightface` and `speaker-recognition` backends.
-
-Shipped in [PR #10441](https://github.com/mudler/LocalAI/pull/10441).

docs/content/blog/2026-06-30-concurrent-scoring-and-pii-ner.md

@@ -1,15 +0,0 @@
-+++
-title = "Concurrent scoring and PII NER on llama.cpp"
-date = 2026-06-30
-description = "Score and TokenClassify now ride llama.cpp's server task queue instead of locking the context, so they run alongside chat traffic."
-url = "/blog/concurrent-scoring-and-pii-ner/"
-+++
-
-The `Score` primitive (used by the router classifier) and `TokenClassify` (used by the PII NER tier) previously locked the llama.cpp context for the duration of the call. They now ride llama.cpp's server task queue instead.
-
-What changes as a result:
-
-- Scoring and token classification run concurrently with chat, completion and embedding traffic, and with each other.
-- The `known_usecases` restriction that forced dedicated scorer and NER model configs on `llama-cpp` is lifted.
-- Repeated scoring calls reuse the prompt KV cache across candidates.
-- Scoring inputs are no longer capped by the physical batch size.

docs/content/blog/2026-07-05-model-capabilities-endpoint.md

@@ -1,14 +0,0 @@
-+++
-title = "Model capabilities endpoint"
-date = 2026-07-05
-description = "GET /v1/models/capabilities reports what each model can do and which modalities it accepts, so clients stop guessing from backend names."
-url = "/blog/model-capabilities-endpoint/"
-+++
-
-`GET /v1/models/capabilities` is a new endpoint: an additive superset of `/v1/models` that reports each model's `capabilities` alongside its `input_modalities` and `output_modalities` (`text`, `image`, `audio`, `video`).
-
-The practical effect is that a client can decide where to send an attachment by asking the server, instead of pattern-matching on backend names. Modalities are either inferred by LocalAI or declared explicitly in the model config.
-
-Because the endpoint is additive, existing `/v1/models` consumers are unaffected.
-
-See [API discovery]({{% relref "features/api-discovery" %}}#model-capabilities). Shipped in [PR #10687](https://github.com/mudler/LocalAI/pull/10687).

docs/content/blog/2026-07-12-longcat-video-and-avatar-generation.md

@@ -1,16 +0,0 @@
-+++
-title = "LongCat video and avatar generation"
-date = 2026-07-12
-description = "A dedicated CUDA backend for LongCat-Video text/image-to-video and LongCat-Video-Avatar-1.5 speech-driven avatars."
-url = "/blog/longcat-video-and-avatar-generation/"
-+++
-
-LocalAI gains a dedicated CUDA backend for the LongCat family: `LongCat-Video` for text-to-video and image-to-video, and `LongCat-Video-Avatar-1.5` for speech-driven avatars.
-
-Highlights:
-
-- Multi-segment continuation, so a clip can be extended beyond a single generation window.
-- Portrait and recorded-audio inputs wired into Studio.
-- An SDPA CUDA 13 ARM64 build, which makes the backend usable on DGX Spark.
-
-See [Video generation]({{% relref "features/video-generation" %}}) for configuration and the available model entries. Shipped in [PR #10792](https://github.com/mudler/LocalAI/pull/10792).

docs/content/blog/_index.md

@@ -1,17 +0,0 @@
-+++
-title = "Blog"
-weight = 10
-icon = "newspaper"
-alwaysopen = false
-aliases = ["/basics/news/", "/whats-new/"]
-+++
-
-Announcements, release write-ups and feature notes from the LocalAI team.
-
-Full changelogs for every version live on [GitHub Releases](https://github.com/mudler/LocalAI/releases). This page is the narrative archive: what shipped, and why it matters.
-
-{{% notice tip %}}
-Prefer a feed reader? Subscribe to [/blog/index.xml](/blog/index.xml).
-{{% /notice %}}
-
-{{< postlist >}}

docs/content/features/text-generation.md

@@ -918,6 +918,200 @@ options:
 The full list of registered parsers lives in `sglang.srt.function_call`
 and `sglang.srt.parser.reasoning_parser`.
 
+### vllm.cpp
+
+[vllm.cpp](https://github.com/mudler/vllm.cpp) is the LocalAI team's C++ port of
+vLLM: the same continuous-batching scheduler, paged KV cache and prefix caching,
+with no Python at inference time. It consumes either a HuggingFace safetensors
+model directory or a `.gguf` file, and applies the model's chat template,
+tool-call parsing and reasoning split engine-side.
+
+#### Setup
+
+```yaml
+name: vllm-cpp
+backend: vllm-cpp
+parameters:
+  model: "Qwen/Qwen3-4B"
+context_size: 8192
+template:
+  use_tokenizer_template: true
+```
+
+#### Configuring the engine with `engine_args`
+
+The same `engine_args:` map the vLLM and SGLang backends accept is honoured
+here, with keys spelled exactly as vLLM's own CLI flags - so a `speculative_config`
+or `kv_transfer_config` block written for vLLM works verbatim. Unknown keys are
+ignored rather than fatal; the engine validates the documents it is handed and
+reports a precise error at load.
+
+```yaml
+name: qwen35-a3b
+backend: vllm-cpp
+parameters:
+  model: "Qwen/Qwen3.5-A3B"
+context_size: 16384
+template:
+  use_tokenizer_template: true
+engine_args:
+  # KV cache sizing: num_blocks * block_size tokens of cache.
+  block_size: 32
+  num_blocks: 1024
+  # Concurrency and the per-step chunked-prefill token budget.
+  max_num_seqs: 32
+  max_num_batched_tokens: 8192
+  # Automatic prefix caching. Omit to keep the model's own default
+  # (on for dense models, off for hybrid / attention-free ones).
+  enable_prefix_caching: true
+  # Scheduler admission order: fcfs (default), priority, or lpm
+  # (cache-aware longest-prefix-match; needs prefix caching to have any effect).
+  scheduling_policy: lpm
+```
+
+| Key | Meaning | Default |
+|-----|---------|---------|
+| `block_size` | KV-cache block size, in tokens per block | 32 |
+| `num_blocks` | KV-cache blocks to allocate | 256 |
+| `max_model_len` | Max sequence length; also settable as `context_size` / `max_model_len` | model config |
+| `max_num_seqs` | Max concurrent sequences the scheduler admits | 8 |
+| `max_num_batched_tokens` | Per-step chunked-prefill token budget | per-arch (2048 dense, 4096/8192 MoE) |
+| `enable_prefix_caching` | Automatic prefix caching; `enable_radix_attention` is an accepted alias | model default |
+| `enable_jump_forward` | Jump-forward decoding, which emits grammar-forced tokens without a model step. Only affects constrained requests (`grammar`, JSON schema) | off |
+| `scheduling_policy` | `fcfs`, `priority`, or `lpm` | `fcfs` |
+| `tool_parser` / `reasoning_parser` | Force a parser instead of chat-template auto-detection | auto |
+| `tokenizer_config` | Override the `tokenizer_config.json` the chat template is read from | `<model_dir>/tokenizer_config.json` |
+| `speculative_config` | Speculative decoding (see below) | disabled |
+| `kv_transfer_config` | External KV connector / LMCache (see below) | none |
+
+Raising `max_num_batched_tokens` lets more prefill land in a single step, at the
+cost of decode latency for requests queued behind it. The default deliberately
+does not scale with `max_num_seqs`, which is what keeps a large concurrent
+prefill from blowing up the per-step activation on the hybrid architectures.
+
+`enable_prefix_caching` and `enable_jump_forward` are tri-state at the engine
+boundary: omitting the key defers to a default (the model's own capability for
+prefix caching, an environment variable for jump forward), while an explicit
+`false` forces the feature off. Those are genuinely different - prefix caching
+defaults *on* for dense models - so write the key only when you mean to override.
+
+#### Speculative decoding
+
+`speculative_config:` takes the same JSON object as vLLM's
+`--speculative-config`. Three methods are supported.
+
+> **Architecture limit.** At the current engine pin, `mtp` and `dflash` are
+> **Qwen3.5 / Qwen3.6 only**. The engine builds a widened speculative KV cache
+> directly for those families rather than through the model registry, so a
+> speculative config on any other architecture (Llama, GLM, Gemma, Mistral, ...)
+> will not work regardless of checkpoint format. `ngram` needs no draft weights
+> and is not subject to this limit.
+
+> **Format support.** `mtp` and `dflash` now work from a `.gguf` target as well
+> as safetensors. An MTP head is read from the GGUF's `nextn.*` tensors when the
+> file declares `<arch>.nextn_predict_layers`; a GGUF exported WITHOUT the head
+> (converted with `--no-mtp`, or predating llama.cpp's Qwen3.5 MTP support) is
+> refused at load naming that as the reason. A DFlash draft may itself be a
+> `dflash`-arch GGUF, and the target may be a GGUF too. `ngram` needs no draft
+> weights and works on any format.
+
+**MTP** (Multi-Token Prediction) uses a draft head shipped inside the target
+checkpoint's own `mtp.*` tensors, so there is no second model to download. It
+requires a **safetensors** checkpoint - the `mtp.*` tensors do not survive GGUF
+conversion, and an MTP config over a `.gguf` model is rejected at load.
+
+```yaml
+engine_args:
+  speculative_config:
+    method: mtp
+    # Optional; defaults to the checkpoint's own head depth, which is
+    # usually the right value. Must be a multiple of that depth.
+    num_speculative_tokens: 1
+```
+
+**DFlash** uses a separate block-diffusion drafter that proposes a whole block
+of tokens in one non-autoregressive forward pass. Unlike MTP, the draft is its
+own checkpoint, so `model:` is **required**:
+
+```yaml
+engine_args:
+  speculative_config:
+    method: dflash
+    model: z-lab/Qwen3.6-27B-DFlash
+    num_speculative_tokens: 4
+```
+
+The draft shares the *target's* `embed_tokens` and `lm_head`, so both must come
+from the same model family and the target must be safetensors.
+
+**The engine does not download the draft.** `model:` is resolved, in order,
+as a path as given, then as the last path segment under LocalAI's models
+directory (`z-lab/Qwen3.6-27B-DFlash` → `<models>/Qwen3.6-27B-DFlash`, which is
+what LocalAI's own downloader produces), then as the whole reference under the
+models directory. Install the draft into LocalAI first, or give an absolute path
+to a directory containing `config.json`. If none of those resolve, the load
+fails immediately naming every location that was tried, rather than reporting a
+missing checkpoint from inside the engine.
+
+**N-gram** needs no draft model at all - it proposes from the prompt's own
+suffix history. `num_speculative_tokens` is required:
+
+```yaml
+engine_args:
+  speculative_config:
+    method: ngram
+    num_speculative_tokens: 4
+    prompt_lookup_min: 5
+    prompt_lookup_max: 5
+```
+
+> **Auto-configuration on import.** When you import a safetensors repository
+> with `backend: vllm-cpp`, LocalAI reads the checkpoint's `config.json` and, if
+> it declares an MTP head (`mtp_num_hidden_layers`), writes
+> `speculative_config: {method: mtp}` into the generated `engine_args` for you.
+> An explicit `speculative_config` in your own config is never overwritten.
+> Importing a DFlash *draft* repository is refused with a warning: a drafter
+> cannot serve on its own, so import the target model and point
+> `speculative_config.model` at the draft.
+
+#### External KV cache with LMCache
+
+`kv_transfer_config:` takes vLLM's `--kv-transfer-config` JSON and selects an
+external KV-cache connector. The `lm://` LMCache client lets prefill KV be
+stored to and reloaded from a shared `lmcache.v1.server`, so a prefix computed
+by one replica does not have to be recomputed by the next:
+
+```yaml
+engine_args:
+  kv_transfer_config:
+    kv_connector: LMCacheConnector
+    kv_role: kv_both          # required whenever kv_connector is set
+    kv_connector_extra_config:
+      host: 127.0.0.1
+      port: 65432
+```
+
+`kv_role` is one of `kv_producer` (store only), `kv_consumer` (load only), or
+`kv_both`. An unregistered connector name, a missing role, or a malformed
+document fails the load with an explicit error rather than silently running
+without the cache.
+
+#### Legacy `options:` list
+
+Earlier versions configured this backend through the flat `options:` list, and
+those configs keep working. Every key in the table above is still read from
+there in `key:value` form, and `engine_args` wins on any key set in both:
+
+```yaml
+options:
+  - max_num_seqs:32
+  - enable_prefix_caching:true
+```
+
+New configs should prefer `engine_args:`, which is the only place the nested
+`speculative_config` / `kv_transfer_config` documents can be written naturally
+rather than as a single-line JSON string.
+
 ### Transformers
 
 [Transformers](https://huggingface.co/docs/transformers/index) is a State-of-the-art Machine Learning library for PyTorch, TensorFlow, and JAX.

gallery/index.yaml

@@ -189,7 +189,7 @@
   files:
     - filename: DeepSeek-V4-Flash-0731-MXFP4.gguf
       uri: huggingface://ggml-org/DeepSeek-V4-Flash-0731-GGUF/DeepSeek-V4-Flash-0731-MXFP4.gguf
-      sha256: c8b46876c3939a6e141f9e4d4aa422981df4a9b84f19e9bb4e1c9a28be31e484
+      sha256: 65f73494afaf27d3add0751a5b716dd2d3e012c66ae0dbbcc1bf8477f92b3ab7
 - name: instella-moe-16b-a3b-think
   url: github:mudler/LocalAI/gallery/virtual.yaml@master
   urls:
@@ -311,7 +311,7 @@
   files:
     - filename: llama-cpp/models/Parable-Granite-4.1-3B-Claude-Fable-5-Q4_K_M/Parable-Granite-4.1-3B-Claude-Fable-5-GGUF-Q4_K_M.gguf
       uri: https://huggingface.co/AnkitAI/Parable-Granite-4.1-3B-Claude-Fable-5-GGUF/resolve/main/Parable-Granite-4.1-3B-Claude-Fable-5-GGUF-Q4_K_M.gguf
-      sha256: 67dc7695d92939c713165761f115c9d892fdff74fcbd987c8bb453b9b8ab645d
+      sha256: dbf202638af23e72508d8316577655d24ba2037fda51ce802b8996977e290bce
 - name: "parable-qwen3-4b-claude-fable-5"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -345,7 +345,7 @@
   files:
     - filename: llama-cpp/models/Parable-Qwen3-4B-Claude-Fable-5-Q4_K_M/Parable-Qwen3-4B-Claude-Fable-5-GGUF-Q4_K_M.gguf
       uri: https://huggingface.co/AnkitAI/Parable-Qwen3-4B-Claude-Fable-5-GGUF/resolve/main/Parable-Qwen3-4B-Claude-Fable-5-GGUF-Q4_K_M.gguf
-      sha256: c94b06a912aa901f3da5689754577ad534415efafc50dcee3f389594a153bf38
+      sha256: 65cc4824fb78ecaf55afdfcdb6dd2e27e1aa805d289db89eae94d32d450403f0
 - name: "parable-granite-4.1-8b-claude-fable-5"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -381,7 +381,7 @@
   files:
     - filename: llama-cpp/models/Parable-Granite-4.1-8B-Claude-Fable-5-Q4_K_M/Parable-Granite-4.1-8B-Claude-Fable-5-GGUF-Q4_K_M.gguf
       uri: https://huggingface.co/AnkitAI/Parable-Granite-4.1-8B-Claude-Fable-5-GGUF/resolve/main/Parable-Granite-4.1-8B-Claude-Fable-5-GGUF-Q4_K_M.gguf
-      sha256: 61a8133c344a0d0a00188395afe33c803e3b973cb4bbfd5ef1fa7110e80bc1c3
+      sha256: 57e464ae3d35253d4351639757dc35e71bab8324d12d49a5870695ce73dc19cf
 - name: "parable-qwen3-8b-claude-fable-5"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -415,7 +415,7 @@
   files:
     - filename: llama-cpp/models/Parable-Qwen3-8B-Claude-Fable-5-Q4_K_M/Parable-Qwen3-8B-Claude-Fable-5-GGUF-Q4_K_M.gguf
       uri: https://huggingface.co/AnkitAI/Parable-Qwen3-8B-Claude-Fable-5-GGUF/resolve/main/Parable-Qwen3-8B-Claude-Fable-5-GGUF-Q4_K_M.gguf
-      sha256: 956070afc8023b8665fe450842f7be76b505b53d142460fd9b588222f4e16112
+      sha256: 4532d2379d38a37279866a030e51d419561f9d4d22fee00d2a33647d66f05065
 - &pocket-35b
   name: "pocket-35b"
   variants:
@@ -2009,7 +2009,7 @@
   files:
     - filename: ds4flash.gguf
       uri: https://huggingface.co/unsloth/DeepSeek-V4-Flash-GGUF
-      sha256: 856c407993ccffa9ad52e23fbef8bb7b458c792a52278f4ca7931741b0c20ce2
+      sha256: 1bfdafd1c288eb1b2bcb629ee9e1b7567dcf0abbe4d20995905a3c3465e9bd1e
 - name: "qwopus3.6-35b-a3b-coder-mtp"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:

website/content/blog/what-landed-in-localai-3-10.md

@@ -0,0 +1,59 @@
+---
+title: "LocalAI 3.10: the Anthropic and Responses APIs, and one image for every GPU"
+date: 2026-01-18
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "anthropic", "open-responses", "gpu", "moonshine"]
+summary: "A /v1/messages endpoint that Claude clients can talk to unchanged, Open Responses compatibility that passes the official acceptance tests, and GPU libraries moved inside the backend containers so one image works on any hardware."
+extracss: ["blog.css"]
+---
+
+Half the tooling worth using speaks a shape of API that is not OpenAI's. You find a client you like, it talks to Anthropic, and swapping it onto a local model means either rewriting the client or gluing a translation layer in front of it. Same story with the agent frameworks that went all in on the Responses API.
+
+3.10.0 adds both surfaces natively, so the client does not have to know.
+
+## Two more front doors
+
+The Anthropic Messages API is served at `/v1/messages`, and at `/messages` for clients that do not prefix. Tool calling, streaming and non-streaming all work, so `anthropic-sdk-go`, LangChain and anything else built on that shape can be pointed at your instance without a code change.
+
+The Open Responses API is at `/v1/responses`, with `/v1/responses/:id` to fetch one and `/v1/responses/:id/cancel` to stop it. It is stateful: pass a `response_id` and the conversation resumes, set `background: true` and the agent runs asynchronously while you go and do something else, then come back for the result. Streaming covers tools, images and audio.
+
+That one passes the [official acceptance tests](https://www.openresponses.org/compliance), which was the bar I wanted to hit before shipping it.
+
+## One image for every GPU
+
+This is the change most likely to affect you even if you do not care about agents.
+
+GPU libraries (CUDA, ROCm, Vulkan) now live inside the backend containers rather than in the image you pull. There is no longer a CUDA image, a ROCm image and a CPU image to choose between. You pull the image, and acceleration works if the hardware is there! Vulkan arm64 builds are in too.
+
+It is experimental, and I want to be clear about that rather than bury it. It is a real architectural change to how every backend gets its libraries, and there will be hardware combinations we did not hit. If it does not work on yours, please file an issue, that is genuinely the most useful thing you can do for this one.
+
+## Everything else
+
+The backend gallery is system aware now, so it only lists backends your machine can actually run. No more scrolling past MLX entries on a Linux box.
+
+Tool calls stream properly, including partial arguments as `input_json_delta`, and models that emit tools as XML (`<function>...</function>`) get parsed instead of dumping the markup into the message text. Both work across llama.cpp, vLLM and diffusers.
+
+Thinking tags are extracted into a separate `reasoning` field rather than being left in the answer, in both SSE and non-SSE mode. The chat UI shows them under a Thinking tab.
+
+There is a video generation page in the web UI with LTX-2 behind it, doing text-to-video and image-to-video with the usual `fps`, `num_frames` and `guidance_scale` controls.
+
+There is request tracing now. `GET /api/traces` returns in-memory request and response logs, `/api/traces/clear` empties them. It is memory backed and drops old entries past a size cap, so it is for debugging an agent that is misbehaving right now, not for an audit trail.
+
+Two new speech backends. Moonshine is an ONNX transcription engine aimed at low-end hardware, and it is the one to reach for on a Pi or an old laptop. It is quick! Pocket-TTS does lightweight TTS with voice cloning, though the cloning path needs a HuggingFace login and a registered voice model, so it is not quite copy-paste.
+
+## Old hardware, and AMD memory
+
+Two fixes worth calling out because they were silent failures rather than errors.
+
+LocalAI was crashing on Intel CPUs without BMI2 (Sandy Bridge, Ivy Bridge), showing up as an `EOF` during model warmup rather than anything that pointed at the cause. It now falls back to `llama-cpp-fallback` on those chips.
+
+On AMD, used and total VRAM were swapped when parsing `rocm-smi` output, so a dual-Radeon box reported nonsense. `HIP_VISIBLE_DEVICES` is also handled properly now, which matters if you are pinning to the discrete GPU.
+
+## Thanks
+
+Thanks to @richiejp, @majiayu000, @nanoandrew4, @DEVMANISHOFFL, @coffeerunhobby, @rampa3, @Nold360, @jroeber and @Divyanshupandey007 for the work in this cycle.
+
+If the unified GPU backends misbehave on your setup, open an issue with what hardware you are on. And if you are wiring up the Anthropic or Responses endpoints and something does not match the spec, tell me, I would rather hear it from you than find out later.
+
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v3.10.0).

website/content/blog/what-landed-in-localai-4-0.md

@@ -0,0 +1,69 @@
+---
+title: "LocalAI 4.0: agents in the core, and a React interface"
+date: 2026-03-14
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "agents", "agenthub", "mcp", "react", "webrtc"]
+summary: "Native agent orchestration with the Agenthub, a rewritten interface with Canvas mode, MCP Apps with tool streaming, and two things removed."
+extracss: ["blog.css"]
+---
+
+Running an agent locally has meant running two things: an inference server, and a separate orchestrator that talks to it. That is a lot of moving parts for something you wanted to try on a Tuesday evening.
+
+4.0.0 puts the agent side in the core. You create agents, give them memory and skills, connect them to MCP servers, and start and stop them from the same interface you already use for models.
+
+This is a major version bump, so there are two removals near the bottom of this post. Read those before you upgrade.
+
+## Agents, and the Agenthub
+
+Agents are managed through the React interface: create one, wire up MCP servers and skills, connect it to Slack, watch what it is doing through a new Events column in the agents list.
+
+Memory has two options. Hybrid search backed by PostgreSQL if you already run one, or in-memory storage via Chromem if you do not want another service. Skills live in a central database rather than being pasted per agent.
+
+The bit I am most curious to see used is [Agenthub](https://agenthub.localai.io), a community space for sharing agent configurations. You publish one, somebody else imports it into their instance and runs it against their own models on their own hardware!
+
+## The interface is React now
+
+The web interface has been rewritten. The old one had reached the point where adding anything meant fighting it.
+
+Canvas mode is the new thing worth turning on: enable it in chat and code blocks and artifacts the model produces render in a preview pane on the right instead of scrolling past you as text. The System view splits Models and Backends into tabs. Traces render as accordions, which makes a long one readable. And if you try to install a model whose weights exceed your system RAM, you get a warning first rather than a locked-up machine.
+
+## MCP Apps
+
+Client-side MCP support is complete in this release ([#8947](https://github.com/mudler/LocalAI/pull/8947)). You pick which MCP servers to enable for a chat directly in the interface, and their tools get injected into the normal chat with streaming, so there is no separate agent mode to switch into.
+
+If you would rather not have any of it, `LOCALAI_DISABLE_MCP` turns the whole thing off.
+
+## Audio, video, and MLX across machines
+
+WebRTC is wired into the Realtime API and the Talk page ([#8790](https://github.com/mudler/LocalAI/pull/8790)), which is a real improvement for latency over what was there before.
+
+Three new audio backends: fish-speech, ace-step.cpp, and faster-qwen3-tts (CUDA only). TTS gained `sample_rate` support through post-processing, and Qwen TTS handles multiple voices.
+
+There is also an experimental MLX distributed backend for spreading a workload across Apple machines ([#8801](https://github.com/mudler/LocalAI/pull/8801)). It is early, so expect rough edges if you try it.
+
+## Infrastructure
+
+Persistent data now has its own location, separate from configuration. `LOCALAI_DATA_PATH` (or `--data-path`) points at where agents, skills, tasks, jobs and the collection database live, defaulting to `data/` under the base path. If you are mounting volumes, this is the one to look at.
+
+Shell completion scripts generate for bash, zsh and fish. There is dedicated Podman documentation now, including rootless setup.
+
+## Two things are gone
+
+The HuggingFace backend has been removed.
+
+AIO images are dropped. They existed to bundle a preset of models with the runtime, and maintaining them across every hardware variant stopped being worth what they gave people. Use the main images and install models from the gallery.
+
+## One known issue
+
+The `diffusers` backend is not in this release. It failed to build because we exhausted our CI limits, so the previous version is still what you get if you install it.
+
+This is an infrastructure problem, not a code one, and it is the kind of thing that will keep happening to us. If you know anybody at GitHub who could help us get better ARM runners, please reach out, I am not too proud to ask.
+
+## Thanks
+
+Thanks to @richiejp, @nanoandrew4, @Weathercold, @sozercan, @lukasdotcom, @loryanstrant, @bittoby and @attilagyorffy.
+
+If you build an agent worth sharing, put it on the Agenthub. The more the merrier!
+
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.0.0).

website/content/blog/what-landed-in-localai-4-1.md

@@ -0,0 +1,76 @@
+---
+title: "LocalAI 4.1: more than one box, and more than one user"
+date: 2026-04-02
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "distributed", "auth", "oidc", "quotas", "fine-tuning"]
+summary: "Distributed cluster mode that places requests by real free VRAM, OIDC with per-user API keys and quotas, and LoRA fine-tuning that exports straight to GGUF."
+extracss: ["blog.css"]
+---
+
+Two problems show up the moment LocalAI stops being a thing you run for yourself.
+
+The first is that you have more than one machine, and only one of them is doing any work. The second is that other people are using your instance, and you have no way to tell who is burning the GPU, or to stop them.
+
+4.1.0 is mostly about those two.
+
+## Running as a cluster
+
+Distributed mode lets you point several nodes at one control plane and stop thinking about which one to call.
+
+Routing orders nodes by available VRAM, so the request lands on the card with room for it. Node groups let you pin models to a subset of the cluster, which is how you keep a heavy diffusion model off the boxes doing embeddings. There is a min/max autoscaler with a reconciler managing node lifecycle, and you can drain a node for maintenance and resume it later through the API instead of pulling it out from under in-flight requests.
+
+Model transfer between nodes goes over S3 or peer to peer, so a model you have already pulled once does not have to come down from the internet again on every node!
+
+The cluster status shows up on the home page.
+
+## Users, keys and quotas
+
+LocalAI ships a multi-user platform now, which is the piece that makes it deployable for a team or a classroom rather than just for you.
+
+- User management from the React interface.
+- OIDC/OAuth against your own identity provider (Google, Keycloak, Authentik, whatever you already run).
+- Invite mode, so registration is closed unless an admin lets somebody in.
+- Per-user API keys.
+- Admin impersonation, for when somebody reports a bug you cannot reproduce.
+
+On top of that there is a quota system: set per-user limits and have them enforced, with a usage dashboard broken down per user and a predictive view of where consumption is heading.
+
+## Fine-tuning without leaving the interface
+
+Both of these are experimental. I would use them on something you can afford to throw away.
+
+Fine-tuning uses HuggingFace TRL to train LoRA adapters, exports the result to GGUF automatically, and imports it back into LocalAI so you can serve what you just trained without moving files around by hand. There is a small evals framework included to check whether the thing you trained is actually better.
+
+The quantization backend produces optimized variants of a model on the fly.
+
+## Agents from the terminal
+
+You can run an agent without the server now:
+
+```sh
+local-ai agent run <name>
+local-ai agent list
+```
+
+`run` takes an agent from the pool registry in `pool.json`, or a single-turn `--prompt` if you just want one answer. Tool calls stream in real time, and the interleaved-thinking bug that mangled output when a model reasoned mid-tool-call is fixed.
+
+## The rest of the interface work
+
+The model pipeline editor is visual, so wiring models together no longer means editing YAML. Backend logs can be scoped to a single model rather than reading the whole stream. Studio pages remember past generations, so images and audio you made last week are still there. The model and backend selectors are searchable. Error toasts link straight to the trace that produced them.
+
+## Under the hood
+
+Inference defaults are pulled from Unsloth and applied across all endpoints and gallery models, so models arrive with sane sampling parameters instead of whatever the default happened to be. `min_p` is supported. When native tool-call parsing fails, an iterative fallback parser takes over rather than returning nothing.
+
+Repeated log lines get collapsed. NVIDIA Jetson and Tegra are detected as first-class platforms. SYCL backends auto-disable `mmap`, which was crashing them on Intel GPUs. llama.cpp bundles `libdl`, `librt` and `libpthread` for portability. And the downloader rewrites HuggingFace URIs through `HF_ENDPOINT`, which is the one you need if you are behind a corporate mirror.
+
+## Thanks
+
+Thanks to @richiejp for a large chunk of this cycle, and to @tv42, @walcz-de, @majiayu000 and @ER-EPR.
+
+There is a full setup walkthrough on video if you would rather watch than read: [youtube.com/watch?v=cMVNnlqwfw4](https://www.youtube.com/watch?v=cMVNnlqwfw4).
+
+If you are setting up distributed mode or OIDC and hit a wall, reach out, I am happy to help you get it standing up.
+
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.1.0).

website/content/blog/what-landed-in-localai-4-2.md

@@ -1,21 +1,20 @@
-+++
-title = "LocalAI 4.2.0: who spoke when, and whose face is that"
-date = 2026-05-11
-description = "Speaker diarization, voice and face recognition, and an Ollama-compatible API."
-url = "/blog/localai-4-2-0/"
-+++
-
-![Diarization: segment, embed, and cluster into speaker-labelled segments](/images/diagrams/diarization-pipeline.png)
+---
+title: "LocalAI 4.2: who spoke when, and whose face is that"
+date: 2026-05-11
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "diarization", "voice-recognition", "face-recognition", "ollama", "backends"]
+summary: "A /v1/audio/diarization endpoint, voice and face recognition with liveness, a drop-in Ollama API, and eleven new backends."
+extracss: ["blog.css"]
+---
 
 You record an hour of standup, run it through Whisper, and get back one long wall of text. Every word is correct. You still have no idea who said any of them, so you end up scrubbing through the audio with the transcript open in another window, guessing at voices.
 
 4.2.0 is mostly about that class of problem. Audio and images carry more than "here are the words" or "here is a picture", and until now LocalAI had nowhere to put the rest of it.
 
-Enough chitchat, let's look at what's in it.
-
 ## Who spoke when
 
-There's a new `/v1/audio/diarization` endpoint, shaped like `/v1/audio/transcriptions` so your existing multipart code mostly carries over:
+There is a new `/v1/audio/diarization` endpoint, shaped like `/v1/audio/transcriptions` so your existing multipart code mostly carries over:
 
 ```bash
 curl http://localhost:8080/v1/audio/diarization \
@@ -37,13 +36,13 @@ curl http://localhost:8080/v1/audio/diarization \
 }
 ```
 
-Two backends serve it. [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) does pure diarization (pyannote-3.0 segmentation, a speaker-embedding extractor, then clustering) and never transcribes, so you don't pay for ASR you didn't ask for. `vibevoice-cpp` emits speaker-labelled segments as a by-product of its long-form ASR pass, so with `include_text=true` you get a transcript per segment for free! `response_format` gives you `json`, `verbose_json`, or `rttm` if you want to feed the output to `dscore`.
+Two backends serve it. [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) does pure diarization (pyannote-3.0 segmentation, a speaker-embedding extractor, then clustering) and never transcribes, so you do not pay for ASR you did not ask for. `vibevoice-cpp` emits speaker-labelled segments as a by-product of its long-form ASR pass, so with `include_text=true` you get a transcript per segment for free! `response_format` gives you `json`, `verbose_json`, or `rttm` if you want to feed the output to `dscore`.
 
 One thing to know before you build on it: `SPEAKER_00` is local to a single request. Run the same meeting twice and the numbering can come out differently, and nothing promises that `SPEAKER_00` in Monday's recording is the same human as `SPEAKER_00` in Tuesday's. If you need identity across files, pair it with `/v1/voice/embed` and keep your own embedding store. Which brings me to..
 
 ## Voices and faces
 
-`/v1/voice/*` is new: verify (are these two clips the same person?), identify (which of my enrolled speakers is this?), embed (give me the vector, I'll do the rest myself), and analyze (age, gender, emotion).
+`/v1/voice/*` is new ([#9500](https://github.com/mudler/LocalAI/pull/9500)): verify (are these two clips the same person?), identify (which of my enrolled speakers is this?), embed (give me the vector, I will do the rest myself), and analyze (age, gender, emotion).
 
 ```bash
 local-ai models install speechbrain-ecapa-tdnn
@@ -63,9 +62,9 @@ curl -sX POST http://localhost:8080/v1/voice/verify \
 
 The default threshold is around 0.25 for ECAPA-TDNN, and it moves per engine, so pass `threshold` explicitly if you swap the model out.
 
-`/v1/face/*` does the same thing for faces, plus detection and demographics, and 4.2.0 adds antispoofing. Holding a printed photo or a phone screen up to the camera is the oldest attack on face auth there is, and the liveness check rejects it.
+`/v1/face/*` does the same thing for faces ([#9480](https://github.com/mudler/LocalAI/pull/9480)), plus detection and demographics, and 4.2.0 adds antispoofing. Holding a printed photo or a phone screen up to the camera is the oldest attack on face auth there is, and the liveness check rejects it.
 
-Some honest limits. Liveness is an arms race and this is not bank-grade. The demographic heads emit confident-looking numbers for age and emotion that you should read as a rough signal and not as a fact about a person. And the default `insightface` buffalo packs are released for non-commercial research use only, so if you're shipping this in a product, pick the OpenCV Zoo entry instead. That's in the docs, but people skip docs, so it's here too.
+Some honest limits. Liveness is an arms race and this is not bank-grade. The demographic heads emit confident-looking numbers for age and emotion that you should read as a rough signal and not as a fact about a person. And the default `insightface` buffalo packs are released for non-commercial research use only, so if you are shipping this in a product, pick the OpenCV Zoo entry instead. That is in the docs, but people skip docs, so it is here too.
 
 The samples never leave your machine, which is the part I actually care about. They go from your process to the backend running next to it and nowhere else. Doing biometrics against somebody else's cloud API always felt like the worst possible trade.
 
@@ -75,21 +74,21 @@ The samples never leave your machine, which is the part I actually care about. T
 OLLAMA_HOST=http://localhost:8080 ollama run qwen3
 ```
 
-LocalAI answers the Ollama API now, so a tool that only ever learned to talk to Ollama keeps working with no code change on your side. `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags`, `/api/show`, `/api/ps` and `/api/version` all land on the engine you were already running, and your existing `/v1/*` clients are untouched.
+LocalAI answers the Ollama API now ([#9284](https://github.com/mudler/LocalAI/pull/9284)), so a tool that only ever learned to talk to Ollama keeps working with no code change on your side. `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags`, `/api/show`, `/api/ps` and `/api/version` all land on the engine you were already running, and your existing `/v1/*` clients are untouched.
 
-There's no `/api/pull` in there. Models come from the LocalAI gallery or from a URL you hand it, so `ollama run` against something you haven't installed yet won't go and fetch it for you.
+There is no `/api/pull` in there. Models come from the LocalAI gallery or from a URL you hand it, so `ollama run` against something you have not installed yet will not go and fetch it for you.
 
-## Video, and a UI repaint
+## Video, and an interface repaint
 
-`stable-diffusion.ggml` generates video now! There are gallery entries for Wan 2.1 FLF2V 14B 720P and Wan i2v 720p, including first-last-frame interpolation.
+`stable-diffusion.ggml` generates video now ([#9420](https://github.com/mudler/LocalAI/pull/9420))! There are gallery entries for Wan 2.1 FLF2V 14B 720P and Wan i2v 720p, including first-last-frame interpolation.
 
-The React UI got a long cycle of work. The chat is redesigned, the palette moved to Nord, and there's i18n across English, Italiano, Español, Deutsch and 简体中文. You can brand your instance too - name, tagline, logo, favicon - and the login page, sidebar, footer and browser tab all pick it up. Handy if you run LocalAI for a team and would rather it didn't look like somebody's side project.
+The React interface got a long cycle of work. The chat is redesigned, the palette moved to Nord, and there is i18n across English, Italiano, Español, Deutsch and 简体中文. You can brand your instance too - name, tagline, logo, favicon - and the login page, sidebar, footer and browser tab all pick it up. Handy if you run LocalAI for a team and would rather it did not look like somebody's side project.
 
 The model config editor is interactive now, with autocomplete over known fields and live validation, and it renames the file on save so you stop accumulating three copies of the same config.
 
 ## Eleven new backends
 
-sglang, ik-llama.cpp, TurboQuant, sam.cpp, Kokoros, qwen3tts.cpp, tinygrad-multimodal (experimental, don't build anything load-bearing on it yet), vibevoice.cpp, LocalVQE, insightface, and voice-rec.
+sglang, ik-llama.cpp, TurboQuant, sam.cpp, Kokoros, qwen3tts.cpp, tinygrad-multimodal (experimental, do not build anything load-bearing on it yet), vibevoice.cpp, LocalVQE, insightface, and voice-rec.
 
 vLLM reached feature parity with llama.cpp in this cycle. The full `AsyncEngineArgs` surface is exposed as a generic YAML map, and tensor-parallel distributed workers let a single model span nodes. There are CUDA 13 builds for vLLM, vLLM-omni and sglang, plus L4T arm64 for Jetson-class boards.
 
@@ -105,16 +104,14 @@ Most of the 279 pull requests here are not features. A sample of what actually w
 - faster-whisper emits word-level timestamps.
 - gfx1151 (Strix Halo / Ryzen AI MAX) works, with `AMDGPU_TARGETS` exposed as a build-arg.
 
-On the security side: an unsafe `sprintf()` came out of the C++ grpc-server, env-supplied API keys are stripped from Settings API requests before they get persisted so they can't leak back out through the config, and deleting a user on PostgreSQL cascades across everything they owned instead of leaving orphaned rows behind.
+On the security side: an unsafe `sprintf()` came out of the C++ grpc-server, env-supplied API keys are stripped from Settings API requests before they get persisted so they cannot leak back out through the config, and deleting a user on PostgreSQL cascades across everything they owned instead of leaving orphaned rows behind.
 
 Distributed mode got a hardening pass. Round-robin across replicas of the same model, "Upgrade All" scoped to the nodes that actually have the backend installed, NATS `backend.upgrade` split off from install, and correct VRAM/RAM reporting on NVIDIA unified-memory hosts.
 
-## Thanks!
+## Thanks
 
 This one had a lot of hands on it. Thanks to @richiejp for the model config editor, Kokoros and a pile of build fixes, @Anai-Guo, @russell, @leinasi2014, @keithmattix for gfx1151, @orbisai0security and @SAY-5 for the security work, @walcz-de, @thelittlefireman, @sec171, @pjbrzozowski, @mvanhorn, @arteven, @Dennisadira, @eglia, @arbrick, @neurocis and @ER-EPR.
 
-If you're wiring up diarization or the voice endpoints and get stuck, open an issue or reach out, I'm genuinely happy to help you get it working. And if LocalAI is useful to you, consider [donating](https://github.com/sponsors/mudler) or just telling somebody about it. The more the merrier!
+If you are wiring up diarization or the voice endpoints and get stuck, open an issue or reach out, I am genuinely happy to help you get it working.
 
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.2.0). See [Speaker diarization]({{% relref "features/audio-diarization" %}}), [Voice recognition]({{% relref "features/voice-recognition" %}}) and [Face recognition]({{% relref "features/face-recognition" %}}).
-
-Cheers!
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.2.0).

website/content/blog/what-landed-in-localai-4-3.md

@@ -0,0 +1,105 @@
+---
+title: "LocalAI 4.3: signed backends, and the prompt cache that was off"
+date: 2026-05-24
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "security", "cosign", "prompt-cache", "distributed", "usage"]
+summary: "Keyless cosign verification for backend OCI images, the llama.cpp prompt cache enabled by default, per-API-key usage attribution, and the replica-pinning bug that kept a second node idle."
+extracss: ["blog.css"]
+---
+
+Here is a gap that had been sitting in LocalAI for a while. The gallery YAML tells LocalAI which OCI image to pull for a backend, and then LocalAI pulls it. Nothing checked that the bytes coming back were the bytes we built. A compromised registry, or somebody in the middle, and you would never know.
+
+4.3.0 closes that, and fixes a default that had been quietly costing everybody a lot of prefill time.
+
+## Signed backends
+
+Every backend image merged by CI is now signed with [sigstore](https://www.sigstore.dev/)/cosign, keyless via Fulcio and Rekor, including each per-arch entry under the manifest list ([#9823](https://github.com/mudler/LocalAI/pull/9823)). It uses OCI 1.1 referrers rather than the legacy `:tag.sig` convention.
+
+On your side, verification runs against a policy that the gallery declares:
+
+```yaml
+verification:
+  issuer_regex: "^https://token\\.actions\\.githubusercontent\\.com$"
+  identity_regex: "^https://github\\.com/mudler/LocalAI/\\.github/workflows/backend_merge\\.yml@.*$"
+  not_before: "2026-05-22T00:00:00Z"
+```
+
+A few details that took some thinking.
+
+`not_before` is the revocation lever. Keyless Fulcio certificates are ephemeral, so there is nothing to revoke on the signing side. Revocation has to be policy side: move the date forward in the gallery YAML and every signature older than it stops validating.
+
+The TUF trusted root is cached process-wide, so installing ten backends from one gallery does one fetch instead of ten.
+
+Digest pinning closes the window between verifying and pulling, which is otherwise a TOCTOU you could drive a truck through.
+
+Strict mode is `--require-backend-integrity`, or `LOCALAI_REQUIRE_BACKEND_INTEGRITY=true`. It turns a missing policy or an empty SHA256 from a warning into a hard failure.
+
+Now the honest part: strict mode is opt-in and off by default, and until a gallery ships a `verification:` block, installs go through with a warning. The default `backend/index.yaml` does not have the blocks populated yet, that is the next step. So today this is machinery that works and is not yet enforcing much. Turn on strict mode in production once your gallery is populated, not before, or you will just break your own installs.
+
+## The prompt cache was off
+
+`llama-cpp` has a server-side prompt cache. LocalAI was not enabling it. So every agent turn, every coding-assistant call, every OpenAI-compatible CLI with a long system prompt, re-prefilled that whole prompt from scratch.
+
+On the reported workload, a repeated system prompt took 5 to 8 minutes per call before this change and seconds after it. Your numbers will depend on how long your prompt is and what hardware you are on.
+
+Two defaults flipped ([#9925](https://github.com/mudler/LocalAI/pull/9925), [#9951](https://github.com/mudler/LocalAI/pull/9951)):
+
+1. `kv_unified` is now `true` in `grpc-server.cpp`. The old `false` was silently force-disabling `cache_idle_slots` at server init, so the host prompt cache got allocated and then never written across requests. That is the one that actually explains the behaviour.
+2. `prompt_cache_all` defaults to `true` at the YAML layer, matching upstream llama.cpp's own default in `common.h`. The per-request `cache_prompt` knob is on out of the box.
+
+You can opt out with `options: ["kv_unified:false"]` or `prompt_cache_all: false`, and there are new keys (`cache_idle_slots`, `checkpoint_every_nt`) if you want to tune it. The model configuration docs got a worked example for the repeated-system-prompt case and an explanation of how `kv_unified`, `cache_ram` and `cache_idle_slots` interact, because they interact in ways that are not obvious.
+
+## Who is burning the GPU
+
+The usage page could tell you how many tokens were spent. It could not tell you who spent them ([#9920](https://github.com/mudler/LocalAI/pull/9920)).
+
+`usage_records` gained a `Source` column (`apikey`, `web`, `legacy`) plus the API key id and name, with an idempotent backfill of older rows on `InitDB`. The auth middleware passes the resolved key and the request source through, and usage middleware snapshots the key id and name at write time, so a key you revoke later still reads correctly in history (it renders as `(revoked)` rather than vanishing).
+
+Two new endpoints:
+
+```
+GET /api/auth/usage/sources        # your own
+GET /api/auth/admin/usage/sources  # everyone, with user_id / api_key_id filters
+```
+
+The admin view truncates at 200 keys. The React usage page gained a Sources tab with a source-mix ribbon, a top-7-plus-Other time chart, and a sortable table. Web interface session traffic is split per user instead of being lumped into one global row.
+
+## Distributed v3, and one good bug
+
+This one is worth writing down because the symptom and the cause were far apart.
+
+An operator reported this:
+
+```
+dgx-spark1     loaded   in_flight=6
+nvidia-thor1   loaded   in_flight=0
+```
+
+Two replicas of the same model, one taking everything, one idle forever. The round-robin was there and looked correct.
+
+The cause: `ModelLoader.Load` cached a `*Model` whose embedded `InFlightTrackingClient` was bound to a single `(nodeID, replicaIndex)`. The first request picked a node and got wrapped. Every request after that reused the wrapper, so it kept going to whichever node won the first pick, even after the reconciler scaled the model out. The routing code was fine. It just was not being consulted again!
+
+`SmartRouter.Route` now runs per request ([#9968](https://github.com/mudler/LocalAI/pull/9968)), the `in_flight ASC, last_used ASC, available_vram DESC` ordering actually fires, and replica selection lives in one place (`PickBestReplica`) with a spec asserting the SQL `ORDER BY` and the Go picker agree on a seeded dataset. `probeHealth` is memoized per `(nodeID, addr)` with a 30 second TTL and `singleflight` coalescing, because llama.cpp serializes `HealthCheck` against in-flight `Predict` and a burst of new requests would otherwise stall on it.
+
+Two other distributed changes.
+
+`POST /api/nodes/:id/backends/install` used to block for up to 3 minutes while the worker pulled the image, which froze the Backends picker in the interface. It returns HTTP 202 and a `jobID` immediately now ([#9928](https://github.com/mudler/LocalAI/pull/9928)). Install and upgrade timeouts are configurable via `LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT` and `LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT`, defaulting to 15 minutes instead of the hardcoded 3. A NATS round-trip timeout while the worker is still pulling reports as `running_on_worker` rather than a hard failure.
+
+Workers also publish debounced install progress (~250ms) that the master forwards into the operations status ([#9958](https://github.com/mudler/LocalAI/pull/9958)), so distributed installs show per-byte progress the same way local ones do. Old workers stay silent and new masters tolerate the silence, so mixed-version clusters keep working.
+
+## Smaller things
+
+`LOCALAI_TRACING_MAX_BODY_BYTES` caps trace payload size, which stops the admin Traces page from trying to render a 40 MB embedding response.
+
+There is a `flake.nix` with a dev shell for NixOS users who do not want to go through Docker.
+
+The `vllm`, `sglang` and `vllm-omni` L4T13 backends are back for Jetson and DGX boxes, switched to PyPI aarch64+cu130 wheels to fix the torch 2.10 ABI mismatch.
+
+A distributed test harness landed in `tests/distributed/`, aimed at catching the class of regression the replica-pinning bug belonged to.
+
+## Thanks
+
+If you run LocalAI in production, the two things to look at here are strict mode (once your gallery has a `verification:` block) and whether the prompt cache change speeds up your workload. I would like to hear numbers from real setups, mine are one data point.
+
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.3.0).