Merge remote-tracking branch 'origin/master' into fix/vllm-cpp-l4t-cuda12-fallback

Two conflicts, both from master moving under a stale branch:

- backend/index.yaml: master rewrote the vllm-cpp gallery description
  (#11363). Keep that rewrite and re-append this branch's paragraph on the
  CUDA 13 / Blackwell-only constraint and the CPU fallback for CUDA 12 hosts.
- backend/go/vllm-cpp/README.md: master gained the ABI pin/mirror note from
  #11386. Keep both sections; they cover different things.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.agents/backend-signing.md

@@ -16,8 +16,7 @@ side (`pkg/oci/cosignverify` plus the gallery YAML).
   per-arch manifest before checking signatures.
 - **Storage:** Signatures are written as OCI 1.1 referrers
   (`--registry-referrers-mode=oci-1-1`) in the new Sigstore bundle format
-  (current cosign releases do this by default; no `--new-bundle-format`
-  flag). No `:sha256-<hex>.sig` tag clutter.
+  (`--new-bundle-format`). No `:sha256-<hex>.sig` tag clutter.
 - **Consumer:** `pkg/oci/cosignverify` discovers the bundle via the
   referrers API, hands it to `sigstore-go`, and verifies it against the
   policy declared in the gallery YAML (`Gallery.Verification`).
@@ -34,14 +33,15 @@ to sign. The job needs:
 
 - `permissions: { id-token: write, contents: read }` at the job level so
   the runner can exchange its GitHub OIDC token for a Fulcio cert.
-- `sigstore/cosign-installer@v3` step (current cosign releases already
-  default to the new bundle format).
+- `sigstore/cosign-installer@v3` step (the pinned cosign v2 release needs
+  `--new-bundle-format` explicitly).
 - After each `docker buildx imagetools create`, resolve the resulting
   list digest with `docker buildx imagetools inspect <tag> --format
   '{{.Manifest.Digest}}'` and sign:
 
 ```sh
 cosign sign --yes --recursive \
+  --new-bundle-format \
   --registry-referrers-mode=oci-1-1 \
   "${REGISTRY_REPO}@${DIGEST}"
 ```
@@ -70,7 +70,7 @@ entry (`backend/index.yaml`):
   url: github:mudler/LocalAI/backend/index.yaml@master
   verification:
     issuer: "https://token.actions.githubusercontent.com"
-    identity_regex: "^https://github\\.com/mudler/LocalAI/\\.github/workflows/backend_merge\\.yml@refs/heads/master$"
+    identity_regex: "^https://github\\.com/mudler/LocalAI/\\.github/workflows/backend_merge\\.yml@refs/(heads/master|tags/.+)$"
     # Optional revocation cutoff; advance during incident response.
     # not_before: "2026-06-01T00:00:00Z"
 ```

.docker/llama-cpp-build-target.sh

@@ -8,8 +8,15 @@ build_type=${2-}
 # ggml-cpu/arch/x86/repack.cpp at -march=sapphirerapids: the job sits on that one
 # translation unit until GitHub kills it at 6h. gcc builds the same file in
 # seconds, so only the SYCL images have to give up the CPU variant matrix.
+#
+# ROCm runs out of the same 6h budget for a different reason: volume, not a
+# stall. hipcc compiles ggml's HIP kernels once per entry in AMDGPU_TARGETS,
+# which is eleven architectures (gfx908 through gfx1201), and the CPU variant
+# matrix lands on top of that. The job built in 2h27m before it was added and
+# has been killed at exactly 6h00m on every run since, so no ROCm llama-cpp
+# image has been published since 2026-08-01.
 case "$build_type" in
-  sycl*)
+  sycl*|hipblas*)
     echo llama-cpp-fallback
     exit 0
     ;;

.github/workflows/backend_merge.yml

@@ -71,8 +71,8 @@ jobs:
 
       # cosign signs each pushed manifest list with --recursive so the
       # index and every per-arch entry get an attached Sigstore bundle.
-      # Recent cosign releases always emit the new bundle format, so
-      # there's no extra CLI flag to opt into it.
+      # The pinned cosign v2 release needs --new-bundle-format explicitly;
+      # the verifier only consumes OCI 1.1 Sigstore bundle referrers.
       - name: Install cosign
         if: github.event_name != 'pull_request'
         uses: sigstore/cosign-installer@v3
@@ -159,6 +159,7 @@ jobs:
           # manifest before checking signatures need the per-arch
           # signatures, not just the list-level one.
           cosign sign --yes --recursive \
+            --new-bundle-format \
             --registry-referrers-mode=oci-1-1 \
             "quay.io/go-skynet/local-ai-backends@${digest}"
 
@@ -185,6 +186,7 @@ jobs:
           ' <<< "$DOCKER_METADATA_OUTPUT_JSON")
           digest=$(docker buildx imagetools inspect "$first_tag" --format '{{.Manifest.Digest}}')
           cosign sign --yes --recursive \
+            --new-bundle-format \
             --registry-referrers-mode=oci-1-1 \
             "localai/localai-backends@${digest}"
 

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=5a8312ef7b8aa7cf14e9a24ac568cabd8725d68a
+AUDIO_CPP_VERSION?=7efbb58def443722ea540d931dd3debee3e4d5e8
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ds4/CMakeLists.txt

@@ -69,7 +69,15 @@ target_include_directories(hw_grpc_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
 
 set(DS4_OBJS "${DS4_DIR}/ds4.o")
 if(DS4_GPU STREQUAL "cuda")
-    list(APPEND DS4_OBJS "${DS4_DIR}/ds4_cuda.o")
+    list(APPEND DS4_OBJS
+        "${DS4_DIR}/ds4_cuda.o"
+        "${DS4_DIR}/cuda/mmq/ds4_ggml_stubs.o"
+        "${DS4_DIR}/cuda/mmq/ds4_mmq.o"
+        "${DS4_DIR}/cuda/mmq/ds4_mmq_d2r.o"
+        "${DS4_DIR}/cuda/mmq/quantize.o"
+        "${DS4_DIR}/cuda/mmq/mmid.o"
+        "${DS4_DIR}/cuda/mmq/mmvq.o"
+        "${DS4_DIR}/cuda/mmq/ds4_repack.o")
 elseif(DS4_GPU STREQUAL "metal")
     list(APPEND DS4_OBJS "${DS4_DIR}/ds4_metal.o")
 elseif(DS4_GPU STREQUAL "cpu")

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=54b36ed9ba42da31b24f2d1a5feb075c2475dbb1
+# Upstream pin lives below as DS4_VERSION?=b0309611041655f4e45671cfd9c9886aff161406
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=54b36ed9ba42da31b24f2d1a5feb075c2475dbb1
+DS4_VERSION?=b0309611041655f4e45671cfd9c9886aff161406
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -23,7 +23,9 @@ CMAKE_ARGS ?= -DCMAKE_BUILD_TYPE=Release
 # are shared by every GPU mode, so append them unconditionally below.
 ifeq ($(BUILD_TYPE),cublas)
     CMAKE_ARGS += -DDS4_GPU=cuda
-    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \
+        cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \
+        cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o
 else ifeq ($(UNAME_S),Darwin)
     CMAKE_ARGS += -DDS4_GPU=metal
     DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
@@ -55,7 +57,7 @@ ds4:
 # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA).
 ds4/ds4.o: ds4
 ifeq ($(BUILD_TYPE),cublas)
-	+$(MAKE) -C ds4 ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+	+$(MAKE) -C ds4 $(DS4_OBJ_TARGET)
 else ifeq ($(UNAME_S),Darwin)
 	+$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=cb9147fd0d9c08a9a84eee5ac405a73f4e10e3e1
+IK_LLAMA_VERSION?=cf1aa57e1a0fabfd015831718fc99d1aec01ada5
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=fcb79282a6bc52e13d858026c42b24fb6e63c97a
+CRISPASR_VERSION?=21901d3f7c23554f072964828363e49ddbc2dc68
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/crispasr/gocrispasr.go

@@ -67,7 +67,16 @@ const defaultTTSSampleRate = 24000
 // resampling, so the WAV header must match it. Returns ok=false for non-piper
 // models (key absent) or an unreadable file, letting the caller fall back to
 // defaultTTSSampleRate.
-func piperSampleRate(modelPath string) (int, bool) {
+func piperSampleRate(modelPath string) (rate int, ok bool) {
+	// A malformed metadata length can make gguf-parser-go panic before it can
+	// return an error. Keep a bad voice file from crash-looping the backend.
+	defer func() {
+		if recover() != nil {
+			rate = 0
+			ok = false
+		}
+	}()
+
 	// Only scalar architecture keys are read, so skip the large array metadata
 	// (phoneme map) and mmap the header - same rationale as pkg/vram's reader.
 	f, err := gguf.ParseGGUFFile(modelPath, gguf.UseMMap(), gguf.SkipLargeMetadata())
@@ -78,7 +87,7 @@ func piperSampleRate(modelPath string) (int, bool) {
 	if !ok || kv.ValueType != gguf.GGUFMetadataValueTypeUint32 {
 		return 0, false
 	}
-	rate := int(kv.ValueUint32())
+	rate = int(kv.ValueUint32())
 	if rate <= 0 {
 		return 0, false
 	}

backend/go/crispasr/gocrispasr_samplerate_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/binary"
+	"math"
 	"os"
 	"path/filepath"
 
@@ -102,6 +103,24 @@ var _ = Describe("piper sample rate", func() {
 			_, ok := piperSampleRate(p)
 			Expect(ok).To(BeFalse())
 		})
+
+		It("returns ok=false instead of panicking on a malformed string length", func() {
+			p := filepath.Join(GinkgoT().TempDir(), "malformed.gguf")
+			var b bytes.Buffer
+			b.WriteString("GGUF")
+			Expect(binary.Write(&b, binary.LittleEndian, uint32(3))).To(Succeed())
+			Expect(binary.Write(&b, binary.LittleEndian, uint64(0))).To(Succeed())
+			Expect(binary.Write(&b, binary.LittleEndian, uint64(1))).To(Succeed())
+			key := "general.name"
+			Expect(binary.Write(&b, binary.LittleEndian, uint64(len(key)))).To(Succeed())
+			b.WriteString(key)
+			Expect(binary.Write(&b, binary.LittleEndian, ggufTypeString)).To(Succeed())
+			Expect(binary.Write(&b, binary.LittleEndian, uint64(math.MaxInt64))).To(Succeed())
+			Expect(os.WriteFile(p, b.Bytes(), 0o644)).To(Succeed())
+
+			_, ok := piperSampleRate(p)
+			Expect(ok).To(BeFalse())
+		})
 	})
 
 	// End-to-end through the built .so. Gated on CRISPASR_PIPER_MODEL_PATH (a

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=db99efdd6d2a43c7937fd55b3359206c680a75b0
+STABLEDIFFUSION_GGML_VERSION?=c6beeef35526c6dc94b74a7fb69f9d2e6a2a7a12
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,30 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=9e1c9025ae61167a3335454d7cc0de6093c21845
+VLLM_CPP_VERSION?=0757cac231ecd571a83c4fd2f50805c9251fc225
+
+# MLX GEMM provider (darwin/metal only; see the metal branch below for why).
+# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun
+# metal`, i.e. a full Xcode the macOS runners do not have, while the wheel ships
+# include/, lib/libmlx.dylib and the compiled mlx.metallib ready to link.
+#
+# DEFAULT ON, but ONLY because VLLM_CPP_VERSION above is pinned at or past
+# vllm.cpp 89c46aeb, which SHAPE-GATES the provider to prefill. The ordering is
+# load-bearing, not incidental:
+#
+#   pin >= 89c46aeb, MLX on   -> 99.1% of MLX-LM  (gated: prefill only)
+#   pin <  89c46aeb, MLX on   -> ~51%             (ungated: it also takes decode)
+#
+# MLX's steel GEMM wins prefill (537 ms TTFT against 602) and loses decode badly,
+# because the provider pays an mx::eval sync plus an output memcpy per call and
+# decode makes ~112 calls per TOKEN. Ungated it does both; gated it does only the
+# good half. So if this pin is ever moved BACKWARDS, this default must go with it.
+VLLM_CPP_MLX?=on
+MLX_VERSION?=0.29.4
+MLX_VENV?=$(abspath ./mlx-venv)
+# Resolved lazily (recursive `=`, not `:=`): the glob only matches once the venv
+# target has run, and the interpreter version in the path varies per runner.
+MLX_ROOT=$(shell echo $(MLX_VENV)/lib/python*/site-packages/mlx)
 
 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
 # server, examples and tests of the engine are never built here.
@@ -49,6 +72,23 @@ else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF
 else ifeq ($(BUILD_TYPE),metal)
 	CMAKE_ARGS+=-DVLLM_CPP_METAL=ON
+	# The optional MLX GEMM provider. vllm.cpp keeps it OFF by default because it
+	# is a ~19 MB libmlx.dylib plus a ~105 MB mlx.metallib, and upstream's
+	# position is that it must earn that cost by measurement. It does, on the
+	# only hardware this build targets: measured on an Apple M4 against the
+	# native MSL GEMM in the SAME binary (arms toggled by
+	# VT_OP_PROVIDER_DISABLE=mlx), Qwen3-1.7B-bf16 p=512 g=128, it is 1.5x to
+	# 2.2x aggregate throughput and 2x to 3x faster TTFT, at equal peak memory
+	# and bit-identical output on every parity shape. See vllm.cpp
+	# docs/BENCHMARKS.md "MLX GEMM provider A/B on Apple M4".
+	#
+	# MLX delegates the dense GEMM ONLY: kPagedAttention stays vllm.cpp's own
+	# kernel, because MLX has no paged-KV primitive at all.
+	#
+	# Set VLLM_CPP_MLX=off for a Metal build without it (smaller image, slower).
+	ifeq ($(VLLM_CPP_MLX),on)
+		MLX_ENABLED=1
+	endif
 else
 	CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF
 endif
@@ -68,10 +108,54 @@ sources/vllm.cpp:
 	git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \
 	git checkout FETCH_HEAD
 
-$(LIB): sources/vllm.cpp
+ifeq ($(MLX_ENABLED),1)
+# A stamp FILE, not a phony target: a phony prerequisite is always "newer" than
+# $(LIB) and would re-link libvllm on every invocation. Keyed on the version so
+# a MLX_VERSION bump reinstalls instead of silently reusing the old wheel.
+MLX_STAMP=$(MLX_VENV)/.mlx-$(MLX_VERSION).stamp
+MLX_CMAKE_ARGS=-DVLLM_CPP_MLX=ON -DMLX_ROOT=$(MLX_ROOT)
+
+$(MLX_STAMP):
+	@if [ ! -x "$(MLX_VENV)/bin/pip" ]; then \
+		python3 -m venv "$(MLX_VENV)" || { echo "vllm-cpp: python3 with venv is required to build the MLX provider; pass VLLM_CPP_MLX=off to build Metal without it" >&2; exit 1; }; \
+	fi
+	"$(MLX_VENV)"/bin/pip install --quiet --disable-pip-version-check "mlx==$(MLX_VERSION)"
+	@# Resolved in the SHELL, not by $(MLX_ROOT): make expands a whole recipe
+	@# before running its first line, so the glob would still be unmatched here.
+	@# Every later use (the cmake args, package.sh) expands after this target has
+	@# completed, where $(MLX_ROOT) does resolve.
+	@root=$$(echo "$(MLX_VENV)"/lib/python*/site-packages/mlx); \
+	test -f "$$root/lib/libmlx.dylib" -a -f "$$root/include/mlx/array.h" || \
+		{ echo "vllm-cpp: mlx==$(MLX_VERSION) did not provide lib/libmlx.dylib + include/mlx/array.h under $$root" >&2; exit 1; }
+	touch $@
+else
+MLX_STAMP=
+MLX_CMAKE_ARGS=
+endif
+
+# govllmcpp.go mirrors vllm.h by hand, and the only guard against the two
+# drifting apart is the vllm_abi_version check inside registerLib - which fires
+# at runtime, on the user's machine, taking down every model load (issue
+# #11379). Compare the two here instead, so moving VLLM_CPP_VERSION past the
+# mirrors turns the build red while the header is still around to diff.
+abi-check: sources/vllm.cpp
+	@engine=$$(sed -n 's/^#define VLLM_ABI_VERSION \([0-9][0-9]*\).*/\1/p' sources/vllm.cpp/include/vllm.h); \
+	backend=$$(sed -n 's/^const abiVersion = \([0-9][0-9]*\).*/\1/p' govllmcpp.go); \
+	if [ -z "$$engine" ] || [ -z "$$backend" ]; then \
+		echo "vllm-cpp: cannot read the ABI version (engine='$$engine' backend='$$backend')" >&2; exit 1; \
+	fi; \
+	if [ "$$engine" != "$$backend" ]; then \
+		echo "vllm-cpp: ABI mismatch: vllm.cpp $(VLLM_CPP_VERSION) is v$$engine, govllmcpp.go mirrors v$$backend." >&2; \
+		echo "  Update the struct mirrors and abiVersion in govllmcpp.go (and the offsets in vllmcpp_test.go) to v$$engine." >&2; \
+		exit 1; \
+	fi; \
+	echo "vllm-cpp: ABI v$$engine matches the pinned engine"
+
+$(LIB): sources/vllm.cpp $(MLX_STAMP)
+	$(MAKE) abi-check
 	mkdir -p build && \
 	cd build && \
-	cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \
+	cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \
 	cmake --build . --config Release -j$(JOBS) --target vllm_shared
 	cp -fL build/$(LIB) ./$(LIB)
 
@@ -79,16 +163,18 @@ vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB)
 	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./
 
 package: vllm-cpp
-	bash package.sh
+	MLX_ROOT="$(MLX_ROOT)" bash package.sh
 
 build: package
 
 clean: purge
-	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp
+	rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp "$(MLX_VENV)"
 
 purge:
 	rm -rf build
 
+.PHONY: abi-check
+
 .NOTPARALLEL:
 
 # The unit specs are pure Go (struct mirrors, option mapping, load

backend/go/vllm-cpp/README.md

@@ -6,7 +6,7 @@ safetensors + GGUF loading, CUDA / CPU / Metal / Vulkan) with no Python at
 inference time.
 
 The backend dlopens the engine's stable C ABI (`libvllm`, `include/vllm.h`,
-ABI v2) through purego:
+ABI v10) through purego:
 
 - `Load` -> `vllm_engine_load`: accepts a `.gguf` file or a HF-style model
   directory (`config.json` + safetensors). `context_size` maps to
@@ -29,6 +29,35 @@ ABI v2) through purego:
   LocalAI's Go-side grammar-constrained tool calling; JSON-schema / regex /
   choice constraints are also exposed by the ABI.
 
+The struct mirrors in `govllmcpp.go` are hand-written against one ABI version,
+and the engine refuses to load against any other. Moving `VLLM_CPP_VERSION` in
+the Makefile therefore means updating `abiVersion` plus the mirrors (and their
+offsets in `vllmcpp_test.go`) in the same change; `make abi-check` compares the
+pinned header against the bindings and the library build runs it first.
+
+## Hardware coverage
+
+The CUDA builds require the CUDA 13 toolchain and target Blackwell only:
+`sm_120a` + `sm_121a` on x86_64, `sm_121a` (GB10 / DGX Spark) on arm64. CUDA
+12.x nvcc cannot compile the Blackwell fp4 kernels, so no CUDA 12 variant is
+shipped and `backend/index.yaml` maps the `nvidia-cuda-12` /
+`nvidia-l4t-cuda-12` capabilities at the CPU build. Practically:
+
+| Host | Installed build |
+|---|---|
+| x86_64 + CUDA 13 | `cuda13-vllm-cpp` |
+| DGX Spark / GB10 (JetPack 7, CUDA 13) | `nvidia-l4t-arm64-vllm-cpp` |
+| Jetson AGX Orin (sm_87, JetPack 6, CUDA 12) | `cpu-vllm-cpp` |
+| Apple Silicon | `metal-vllm-cpp` |
+| Anything else | `vulkan-vllm-cpp` or `cpu-vllm-cpp` |
+
+The capability a host reports comes from `/run/localai/capability` inside the
+LocalAI container, which the image bakes in at build time (see `Dockerfile`).
+A DGX Spark running the CUDA 12 `-nvidia-l4t-arm64` image therefore reports
+`nvidia-l4t-cuda-12` and gets the CPU build; use the `-nvidia-l4t-arm64-cuda-13`
+image, or set `LOCALAI_FORCE_META_BACKEND_CAPABILITY=nvidia-l4t-cuda-13`, to
+get the GPU one.
+
 Model config example:
 
 ```yaml
@@ -41,5 +70,50 @@ options:
 - max_num_seqs:16
 ```
 
+## Apple Silicon: the MLX GEMM provider (ON by default, gated to prefill)
+
+`BUILD_TYPE=metal` builds vllm.cpp's MLX provider for the dense GEMM
+(`VLLM_CPP_MLX=on`, the default here). It is on because upstream now SHAPE-GATES
+it to prefill; it was briefly off in this branch's history, and that was correct
+at the time for an ungated provider.
+
+The gate matters more than the flag. MLX's steel GEMM wins prefill but loses
+decode, because the provider pays an `mx::eval` synchronisation plus an output
+memcpy on every call and decode makes ~112 calls *per token*. Measured on an
+Apple M4, Qwen3-1.7B-bf16 warm at p=512 g=128:
+
+| configuration | prefill TTFT | warm throughput |
+|---|--:|--:|
+| MLX **gated to prefill** (pin >= 89c46aeb) | **524.5 ms** | **24.37 tok/s, 97.6% of MLX-LM** |
+| MLX ungated (older pins) | 537 ms | 12.7 tok/s |
+| MLX off | 602 ms | 23.9 tok/s, 95.9% |
+
+Ratios are against an MLX-LM baseline measured INTERLEAVED with ours over four
+ABBA blocks (its spread 0.34%, ours 0.12%). An earlier revision of this file
+claimed 99.1%; that used a two-run MLX-LM baseline containing an outlier and
+overstated us by about 1.5 points.
+
+**`VLLM_CPP_VERSION` and this flag are coupled.** Moving the pin back before
+`89c46aeb` while leaving `VLLM_CPP_MLX=on` would take the middle row — roughly
+half throughput. If you roll the pin back, roll the default back with it.
+
+One caveat: MLX's GEMM is not bit-identical to the native kernel, so an MLX build
+produces a different greedy sequence than a non-MLX one. That is a property of the
+provider, not of the gate, and it predates this packaging. Full disposition in
+vllm.cpp `docs/BENCHMARKS.md`.
+
+Build knobs:
+
+- `VLLM_CPP_MLX=off` builds Metal without the provider: ~124 MB smaller, and
+  96.4% of MLX-LM instead of 99.1%.
+- `MLX_VERSION` pins the wheel (default `0.29.4`). MLX is consumed as the
+  prebuilt pip wheel because building it from source needs `xcrun metal`, i.e. a
+  full Xcode the macOS runners do not have.
+
+Packaging vendors `libmlx.dylib`, `mlx.metallib` and MLX's MIT license into
+`package/lib/`, and rewrites `libvllm.dylib`'s rpath to `@loader_path/lib`
+(re-signing it, since `install_name_tool` invalidates the signature). The
+metallib must stay beside `libmlx.dylib`: MLX looks for it there.
+
 Testing: `make test` runs the unit specs; export `VLLM_CPP_MODEL=<model>` (and
 optionally `VLLM_CPP_LIBRARY=<libvllm path>`) to enable the e2e specs.

backend/go/vllm-cpp/govllmcpp.go

@@ -1,6 +1,6 @@
 package main
 
-// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v2).
+// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10).
 //
 // The structs below are hand-mirrored PODs of the C declarations, with
 // explicit padding so the Go layout matches the C layout on linux/darwin
@@ -17,15 +17,21 @@ import (
 	"github.com/ebitengine/purego"
 )
 
-// abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h).
-const abiVersion = 5
+// abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h). It must track
+// the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks
+// the two against each other, because a mismatch is only caught at runtime by
+// registerLib, where it takes the backend down on every load (issue #11379).
+const abiVersion = 10
 
 // vllm_status (vllm.h).
 const (
 	vllmOK = 0
 )
 
-// cModelParams mirrors vllm_model_params.
+// cModelParams mirrors vllm_model_params. The fields the backend does not set
+// are still mirrored: the engine reads the whole struct, so the Go value must
+// be the same size as the C one. Every one of them is inert when zeroed, which
+// is what keeps the engine byte-identical to the pre-v6 behavior.
 type cModelParams struct {
 	ModelPath           uintptr // const char*
 	TokenizerConfigPath uintptr // const char*
@@ -35,11 +41,18 @@ type cModelParams struct {
 	MaxNumSeqs          int32
 	ToolParser          uintptr // const char*; NULL = auto-detect (ABI v4)
 	ReasoningParser     uintptr // const char*; NULL = auto-detect (ABI v5)
+	SpeculativeConfig   uintptr // const char*; NULL = no speculation (ABI v6)
+	EnablePrefixCaching int32   // 0 = model default, 1 = on, 2 = off (ABI v7)
+	MaxNumBatchedTokens int32   // <= 0 = per-arch default (ABI v9)
+	SchedulingPolicy    uintptr // const char*; NULL = "fcfs" (ABI v9)
+	KVTransferConfig    uintptr // const char*; NULL = no connector (ABI v9)
+	EnableJumpForward   int32   // 0 = env-resolved (off), 1 = on, 2 = off (ABI v10)
+	_                   [4]byte
 }
 
-// cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields
-// included). Padding matches the C compiler's: the uint64 seed is 8-aligned,
-// and each pointer following an int32 is 8-aligned.
+// cSamplingParams mirrors vllm_sampling_params (structured fields included).
+// Padding matches the C compiler's: the uint64 seed is 8-aligned, and each
+// pointer following an int32 is 8-aligned.
 type cSamplingParams struct {
 	Temperature          float32
 	TopP                 float32
@@ -65,6 +78,10 @@ type cSamplingParams struct {
 	StructuredGrammar    uintptr // const char*
 	StructuredJSONObject int32
 	_                    [4]byte
+	// Per-request custom logits processor (ABI v8). Left NULL: a Go callback
+	// would have to run inside the sampler's decode step for every token.
+	LogitsProcessor         uintptr // vllm_logits_processor; NULL = none
+	LogitsProcessorUserData uintptr // void*, passed back to the callback
 }
 
 // cCompletion mirrors vllm_completion.

backend/go/vllm-cpp/package.sh

@@ -43,6 +43,50 @@ elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
     cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
 elif [ $(uname -s) = "Darwin" ]; then
     echo "Detected Darwin"
+    # Vendor the optional MLX GEMM provider, when libvllm was built against it.
+    # Three facts drive every line below, each verified on an Apple M4 before it
+    # was written:
+    #   1. libvllm.dylib carries an LC_LOAD_DYLIB on @rpath/libmlx.dylib, and its
+    #      build-time LC_RPATH points inside the build venv. That path does not
+    #      exist on a user's machine, so it must become @loader_path/lib.
+    #   2. MLX finds its ~100 MB mlx.metallib beside its OWN dylib, so the two
+    #      files have to land in the same directory or every Metal op dies with
+    #      "Failed to load the default metallib".
+    #   3. install_name_tool invalidates the code signature, and macOS refuses to
+    #      load an arm64 image whose signature does not match, so the patched
+    #      library must be re-signed ad-hoc afterwards.
+    if otool -L "$CURDIR/package/libvllm.dylib" 2>/dev/null | grep -q "libmlx.dylib"; then
+        MLX_LIB_DIR="${MLX_ROOT}/lib"
+        if [ ! -f "$MLX_LIB_DIR/libmlx.dylib" ] || [ ! -f "$MLX_LIB_DIR/mlx.metallib" ]; then
+            echo "Error: libvllm.dylib links libmlx.dylib but $MLX_LIB_DIR is missing libmlx.dylib/mlx.metallib" >&2
+            exit 1
+        fi
+        echo "Vendoring the MLX GEMM provider from $MLX_LIB_DIR"
+        cp -fLv "$MLX_LIB_DIR/libmlx.dylib" "$CURDIR/package/lib/"
+        cp -fLv "$MLX_LIB_DIR/mlx.metallib" "$CURDIR/package/lib/"
+        # MLX is MIT and we redistribute its binaries, so its license ships with
+        # them. mlx-metal is the wheel carrying the dylib and the metallib.
+        MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx_metal-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
+        if [ -z "$MLX_LICENSE" ]; then
+            MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx-*.dist-info/licenses/LICENSE 2>/dev/null | head -1)
+        fi
+        if [ -z "$MLX_LICENSE" ]; then
+            echo "Error: could not find the MLX LICENSE to redistribute alongside libmlx.dylib" >&2
+            exit 1
+        fi
+        cp -fLv "$MLX_LICENSE" "$CURDIR/package/lib/LICENSE.mlx"
+        # Drop every build-tree rpath, then point at the packaged copy.
+        otool -l "$CURDIR/package/libvllm.dylib" | awk '/LC_RPATH/{f=1;next} f&&/ path /{print $2;f=0}' | while read -r rp; do
+            install_name_tool -delete_rpath "$rp" "$CURDIR/package/libvllm.dylib" 2>/dev/null || true
+        done
+        install_name_tool -add_rpath "@loader_path/lib" "$CURDIR/package/libvllm.dylib"
+        codesign -f -s - "$CURDIR/package/libvllm.dylib"
+        # A broken rpath must fail the BUILD, not the user's first inference.
+        if ! otool -l "$CURDIR/package/libvllm.dylib" | grep -q "@loader_path/lib"; then
+            echo "Error: libvllm.dylib did not get the @loader_path/lib rpath" >&2
+            exit 1
+        fi
+    fi
 else
     echo "Error: Could not detect architecture"
     exit 1

backend/go/vllm-cpp/vllmcpp_test.go

@@ -16,10 +16,17 @@ func TestVllmCpp(t *testing.T) {
 	RunSpecs(t, "vllm-cpp suite")
 }
 
-// The Go POD mirrors must match the C struct layout of vllm.h (ABI v2)
+// The Go POD mirrors must match the C struct layout of vllm.h (ABI v10)
 // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
 // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
 var _ = Describe("C ABI struct mirrors", func() {
+	It("declares the ABI version the pinned engine reports", func() {
+		// VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile).
+		// Moving the pin past this without growing the mirrors below ships a
+		// backend that refuses every load at startup (issue #11379).
+		Expect(abiVersion).To(Equal(10))
+	})
+
 	It("cModelParams matches vllm_model_params", func() {
 		var p cModelParams
 		Expect(unsafe.Offsetof(p.ModelPath)).To(Equal(uintptr(0)))
@@ -30,10 +37,16 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.MaxNumSeqs)).To(Equal(uintptr(28)))
 		Expect(unsafe.Offsetof(p.ToolParser)).To(Equal(uintptr(32)))
 		Expect(unsafe.Offsetof(p.ReasoningParser)).To(Equal(uintptr(40)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(48)))
+		Expect(unsafe.Offsetof(p.SpeculativeConfig)).To(Equal(uintptr(48)))
+		Expect(unsafe.Offsetof(p.EnablePrefixCaching)).To(Equal(uintptr(56)))
+		Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60)))
+		Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64)))
+		Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72)))
+		Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80)))
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88)))
 	})
 
-	It("cSamplingParams matches vllm_sampling_params (ABI v2)", func() {
+	It("cSamplingParams matches vllm_sampling_params", func() {
 		var p cSamplingParams
 		Expect(unsafe.Offsetof(p.Temperature)).To(Equal(uintptr(0)))
 		Expect(unsafe.Offsetof(p.TopP)).To(Equal(uintptr(4)))
@@ -55,7 +68,9 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.NStructuredChoice)).To(Equal(uintptr(96)))
 		Expect(unsafe.Offsetof(p.StructuredGrammar)).To(Equal(uintptr(104)))
 		Expect(unsafe.Offsetof(p.StructuredJSONObject)).To(Equal(uintptr(112)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120)))
+		Expect(unsafe.Offsetof(p.LogitsProcessor)).To(Equal(uintptr(120)))
+		Expect(unsafe.Offsetof(p.LogitsProcessorUserData)).To(Equal(uintptr(128)))
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136)))
 	})
 
 	It("cCompletion matches vllm_completion", func() {

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=2ca53bb45e38748d07b310eeb36245a7157ac882
+WHISPER_CPP_VERSION?=306c88f4d1286aec1bf96e544632897886af5501
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/index.yaml

@@ -193,12 +193,27 @@
   alias: "vllm-cpp"
   license: apache-2.0
   description: |
-    vllm.cpp is a from-scratch C++20 port of vLLM created and maintained by the LocalAI team.
-    It mirrors vLLM's V1 architecture (paged KV cache, continuous batching, prefix caching,
-    scheduler, sampler) on a portable tensor runtime with no Python, PyTorch or ggml at
-    inference time. It loads Hugging Face safetensors and GGUF checkpoints, supports
-    structured output (JSON schema / regex / choice / GBNF grammar) enforced in-engine,
-    and runs on CPU, NVIDIA CUDA (Blackwell-family), Apple Metal and Vulkan.
+    ALPHA development builds. Try it, but llama-cpp stays the recommendation for
+    production use.
+
+    vllm.cpp is an Apache-2.0 C++20 inference engine maintained by the LocalAI team,
+    developed in its own repository and usable without LocalAI. It began as a port of
+    vLLM and keeps vLLM as its reference implementation, checking output against it and
+    benchmarking against it, while growing a featureset of its own. It implements vLLM's
+    V1 architecture (paged KV cache, continuous batching, prefix caching, scheduler,
+    sampler) on a portable tensor runtime with no Python, PyTorch or ggml at inference
+    time. It loads GGUF as well as Hugging Face safetensors, supports structured output
+    (JSON schema / regex / choice / GBNF grammar) enforced in-engine, ships speculative
+    decoding and KV offload, and runs on CPU, NVIDIA CUDA (Blackwell-family), Apple
+    Metal and Vulkan.
+
+    The CUDA builds require the CUDA 13 toolchain and target Blackwell only: sm_120a
+    plus sm_121a on x86_64, and sm_121a (GB10 / DGX Spark) on arm64. Older NVIDIA
+    hardware and CUDA 12 hosts - including Jetson AGX Orin (sm_87, JetPack 6) - run
+    the CPU build instead.
+
+    The project is expected to be renamed as it diverges further from vLLM; the new
+    name is still to be decided.
   urls:
     - https://github.com/mudler/vllm.cpp
   tags:
@@ -216,6 +231,12 @@
     nvidia-cuda-13: "cuda13-vllm-cpp"
     nvidia-l4t: "nvidia-l4t-arm64-vllm-cpp"
     nvidia-l4t-cuda-13: "nvidia-l4t-arm64-vllm-cpp"
+    # No CUDA 12 variant exists: 12.x nvcc cannot compile the Blackwell fp4
+    # kernels, so those hosts run the CPU build. Mapped explicitly rather than
+    # left to the "default" catch-all so the fallback is visible here instead
+    # of looking like an oversight.
+    nvidia-cuda-12: "cpu-vllm-cpp"
+    nvidia-l4t-cuda-12: "cpu-vllm-cpp"
 - !!merge <<: *vllm-cpp
   name: "vllm-cpp-development"
   capabilities:
@@ -226,6 +247,8 @@
     nvidia-cuda-13: "cuda13-vllm-cpp-development"
     nvidia-l4t: "nvidia-l4t-arm64-vllm-cpp-development"
     nvidia-l4t-cuda-13: "nvidia-l4t-arm64-vllm-cpp-development"
+    nvidia-cuda-12: "cpu-vllm-cpp-development"
+    nvidia-l4t-cuda-12: "cpu-vllm-cpp-development"
 - &crispasr
   name: "crispasr"
   alias: "crispasr"

core/gallery/backend_index_capabilities_test.go

@@ -0,0 +1,83 @@
+package gallery_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/system"
+)
+
+// loadBackendIndex parses backend/index.yaml once for the whole suite.
+var loadBackendIndex = sync.OnceValues(func() (gallery.GalleryElements[*gallery.GalleryBackend], error) {
+	data, err := os.ReadFile(filepath.Join("..", "..", "backend", "index.yaml"))
+	if err != nil {
+		return nil, err
+	}
+	var entries gallery.GalleryElements[*gallery.GalleryBackend]
+	if err := yaml.Unmarshal(data, &entries); err != nil {
+		return nil, err
+	}
+	return entries, nil
+})
+
+var _ = Describe("backend/index.yaml capability maps", func() {
+	var entries gallery.GalleryElements[*gallery.GalleryBackend]
+
+	BeforeEach(func() {
+		var err error
+		entries, err = loadBackendIndex()
+		Expect(err).ToNot(HaveOccurred())
+		Expect(entries).ToNot(BeEmpty())
+	})
+
+	// A capability pointing at a name that does not exist is invisible until a
+	// host with exactly that capability tries to install: FindBestBackendFromMeta
+	// returns nil and the install fails with "no backend found".
+	It("resolves every capability reference to an entry in the index", func() {
+		names := map[string]struct{}{}
+		for _, e := range entries {
+			names[e.Name] = struct{}{}
+		}
+
+		dangling := []string{}
+		for _, e := range entries {
+			for capability, target := range e.CapabilitiesMap {
+				if _, ok := names[target]; !ok {
+					dangling = append(dangling, fmt.Sprintf("  %s -> %s: %q", e.Name, capability, target))
+				}
+			}
+		}
+		Expect(dangling).To(BeEmpty(), "capabilities naming a missing entry:\n%s", strings.Join(dangling, "\n"))
+	})
+
+	// vllm.cpp's CUDA kernels need the CUDA 13 toolchain (12.x nvcc cannot
+	// compile the Blackwell fp4 paths), so CUDA 12 hosts have no GPU build to
+	// install and must land on the CPU one. Assert the fallback is explicit
+	// rather than an accident of the "default" catch-all, so mapping these
+	// capabilities at a CUDA image later is a test failure and not a host that
+	// pulls kernels it cannot run.
+	DescribeTable("routes vllm-cpp hosts to the build their toolchain supports",
+		func(metaName, capability, expected string) {
+			meta := entries.FindByName(metaName)
+			Expect(meta).ToNot(BeNil())
+
+			resolved := meta.FindBestBackendFromMeta(system.NewCapabilityState(capability), entries)
+			Expect(resolved).ToNot(BeNil())
+			Expect(resolved.Name).To(Equal(expected))
+		},
+		Entry("CUDA 12 x86_64 gets the CPU build", "vllm-cpp", "nvidia-cuda-12", "cpu-vllm-cpp"),
+		Entry("CUDA 12 Jetson (AGX Orin) gets the CPU build", "vllm-cpp", "nvidia-l4t-cuda-12", "cpu-vllm-cpp"),
+		Entry("CUDA 13 Jetson (DGX Spark) gets the L4T build", "vllm-cpp", "nvidia-l4t-cuda-13", "nvidia-l4t-arm64-vllm-cpp"),
+		Entry("CUDA 13 x86_64 gets the CUDA build", "vllm-cpp", "nvidia-cuda-13", "cuda13-vllm-cpp"),
+		Entry("development CUDA 12 Jetson gets the CPU build", "vllm-cpp-development", "nvidia-l4t-cuda-12", "cpu-vllm-cpp-development"),
+		Entry("development CUDA 13 Jetson gets the L4T build", "vllm-cpp-development", "nvidia-l4t-cuda-13", "nvidia-l4t-arm64-vllm-cpp-development"),
+	)
+})

core/gallery/estimate_warm.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/concurrency"
 	"github.com/mudler/LocalAI/pkg/system"
 	"github.com/mudler/LocalAI/pkg/vram"
 	"github.com/mudler/xlog"
@@ -101,7 +102,7 @@ func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemSt
 		return
 	}
 
-	go func() {
+	concurrency.SafeGo(func() {
 		started := time.Now()
 
 		models, err := AvailableGalleryModelsCached(galleries, systemState)
@@ -131,7 +132,7 @@ func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemSt
 
 		for i := 0; i < cfg.Concurrency; i++ {
 			wg.Add(1)
-			go func() {
+			concurrency.SafeGo(func() {
 				defer wg.Done()
 				for m := range cursor {
 					// Per entry, not for the run: one unreachable weight file
@@ -164,7 +165,7 @@ func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemSt
 
 					cancel()
 				}
-			}()
+			})
 		}
 
 	feed:
@@ -183,7 +184,7 @@ func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemSt
 			return
 		}
 		xlog.Info("gallery caches warmed", "estimates", warmed, "variants", warmedVariants, "of", len(models), "took", time.Since(started).Round(time.Second))
-	}()
+	})
 }
 
 // EstimateWarmConfigFromEnv reads the warm-up bounds from the environment,

core/gallery/estimate_warm_test.go

@@ -1,11 +1,20 @@
 package gallery_test
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
+	"math"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"path/filepath"
+	"time"
 
+	gguf "github.com/gpustack/gguf-parser-go"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/gallery"
@@ -57,6 +66,46 @@ var _ = Describe("VRAM estimate warm-up", func() {
 		Consistently(func() bool { return true }, "100ms").Should(BeTrue())
 	})
 
+	It("does not crash the server when remote GGUF metadata is malformed", func() {
+		payload := warmMalformedGGUF()
+		requested := make(chan struct{})
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			select {
+			case <-requested:
+			default:
+				close(requested)
+			}
+			http.ServeContent(w, r, "model.gguf", time.Time{}, bytes.NewReader(payload))
+		}))
+		DeferCleanup(server.Close)
+
+		galleryPath := filepath.Join(state.Model.ModelsPath, "malformed-gallery.yaml")
+		index, err := yaml.Marshal([]gallery.GalleryModel{{Metadata: gallery.Metadata{
+			Name: "malformed-gguf",
+			AdditionalFiles: []gallery.File{{
+				Filename: "model.gguf",
+				URI:      server.URL + "/model.gguf",
+			}},
+		}}})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(os.WriteFile(galleryPath, index, 0600)).To(Succeed())
+
+		cfg := gallery.DefaultEstimateWarmConfig
+		cfg.Limit = 1
+		cfg.Concurrency = 1
+		cfg.Contexts = []uint32{8192}
+		gallery.WarmEstimateCache(context.Background(), []config.Gallery{{
+			Name: "malformed",
+			URL:  "file://" + galleryPath,
+		}}, state, cfg)
+
+		Eventually(requested, "2s").Should(BeClosed())
+		// The warm-up is detached. Give its parser time to consume the response;
+		// before the recovery boundary, that goroutine panicked and killed the
+		// entire test process (and the LocalAI server in production).
+		Consistently(func() bool { return true }, "300ms").Should(BeTrue())
+	})
+
 	Describe("configuration from the environment", func() {
 		AfterEach(func() {
 			os.Unsetenv("LOCALAI_VRAM_WARM_LIMIT")
@@ -113,3 +162,19 @@ var _ = Describe("VRAM estimate warm-up", func() {
 	})
 
 })
+
+func warmMalformedGGUF() []byte {
+	payload := make([]byte, 0, 128)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMagicGGUFLe))
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFVersionV3))
+	payload = binary.LittleEndian.AppendUint64(payload, 0)
+	payload = binary.LittleEndian.AppendUint64(payload, 1)
+	key := "tokenizer.ggml.tokens"
+	payload = binary.LittleEndian.AppendUint64(payload, uint64(len(key)))
+	payload = append(payload, key...)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeArray))
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeString))
+	payload = binary.LittleEndian.AppendUint64(payload, 1)
+	payload = binary.LittleEndian.AppendUint64(payload, math.MaxUint64)
+	return payload
+}

core/gallery/importers/llama-cpp.go

@@ -401,7 +401,10 @@ func maybeApplyMTPDefaults(modelConfig *config.ModelConfig, details Details, cfg
 		}
 	}()
 
-	f, err := gguf.ParseGGUFFileRemote(ctx, probeURL)
+	// MTP markers are architecture scalars. Avoid allocating tokenizer and
+	// other large arrays from an untrusted remote header; panic recovery cannot
+	// contain a fatal out-of-memory condition.
+	f, err := gguf.ParseGGUFFileRemote(ctx, probeURL, gguf.SkipLargeMetadata())
 	if err != nil {
 		xlog.Debug("[mtp-importer] failed to read remote GGUF header for MTP detection", "uri", probeURL, "error", err)
 		return

core/http/middleware/trace.go

@@ -60,6 +60,7 @@ type APIExchange struct {
 }
 
 var traceBuffer *circularbuffer.Queue[APIExchange]
+var inFlightTraces = make(map[string]APIExchange)
 var mu sync.Mutex
 var logChan = make(chan traceCommand, 100)
 var traceIDSeq atomic.Uint64
@@ -126,16 +127,17 @@ func initializeTracing(dataPath string, maxItems int) {
 					continue
 				}
 				exchange := *command.exchange
+				mu.Lock()
+				delete(inFlightTraces, exchange.ID)
+				if traceBuffer != nil {
+					traceBuffer.Enqueue(exchange)
+				}
+				mu.Unlock()
 				if command.store != nil {
 					if err := command.store.Append(exchange.ID, exchange); err != nil {
 						xlog.Warn("Failed to persist API trace", "error", err)
 					}
 				}
-				mu.Lock()
-				if traceBuffer != nil {
-					traceBuffer.Enqueue(exchange)
-				}
-				mu.Unlock()
 			}
 		}()
 	})
@@ -261,6 +263,38 @@ func TraceMiddleware(app *application.Application) echo.MiddlewareFunc {
 			// tens of MB, which then locks the admin Traces UI fetching the
 			// JSON dump faster than the 5s auto-refresh.
 			maxBodyBytes := app.ApplicationConfig().TracingMaxBodyBytes
+			requestHeaders := redactSensitiveHeaders(c.Request().Header)
+			requestBody, requestTruncated := truncateForTrace(body, maxBodyBytes)
+			exchange := APIExchange{
+				ID:        nextTraceID(),
+				Timestamp: startTime,
+				ClientIP:  c.RealIP(),
+				UserAgent: c.Request().UserAgent(),
+				Request: APIExchangeRequest{
+					Method:        c.Request().Method,
+					Path:          c.Path(),
+					Headers:       &requestHeaders,
+					Body:          &requestBody,
+					BodyTruncated: requestTruncated,
+					BodyBytes:     len(body),
+				},
+			}
+			if user := auth.GetUser(c); user != nil {
+				exchange.UserID = user.ID
+				exchange.UserName = user.Name
+			}
+			mu.Lock()
+			inFlightTraces[exchange.ID] = exchange
+			mu.Unlock()
+			queued := false
+			defer func() {
+				if queued {
+					return
+				}
+				mu.Lock()
+				delete(inFlightTraces, exchange.ID)
+				mu.Unlock()
+			}()
 
 			// Wrap response writer to capture body
 			resBody := new(bytes.Buffer)
@@ -287,47 +321,27 @@ func TraceMiddleware(app *application.Application) echo.MiddlewareFunc {
 			// the trace endpoint is admin-only but the buffer is also reachable
 			// via any heap-dump-style introspection, and tokens shouldn't
 			// outlive the request that carried them.
-			requestHeaders := redactSensitiveHeaders(c.Request().Header)
-			requestBody, requestTruncated := truncateForTrace(body, maxBodyBytes)
 			responseHeaders := redactSensitiveHeaders(c.Response().Header())
 			responseBody := make([]byte, resBody.Len())
 			copy(responseBody, resBody.Bytes())
-			exchange := APIExchange{
-				ID:        nextTraceID(),
-				Timestamp: startTime,
-				Duration:  time.Since(startTime),
-				ClientIP:  c.RealIP(),
-				UserAgent: c.Request().UserAgent(),
-				Request: APIExchangeRequest{
-					Method:        c.Request().Method,
-					Path:          c.Path(),
-					Headers:       &requestHeaders,
-					Body:          &requestBody,
-					BodyTruncated: requestTruncated,
-					BodyBytes:     len(body),
-				},
-				Response: APIExchangeResponse{
-					Status:        status,
-					Headers:       &responseHeaders,
-					Body:          &responseBody,
-					BodyTruncated: mw.truncated,
-					BodyBytes:     mw.totalBytes,
-				},
+			exchange.Duration = time.Since(startTime)
+			exchange.Response = APIExchangeResponse{
+				Status:        status,
+				Headers:       &responseHeaders,
+				Body:          &responseBody,
+				BodyTruncated: mw.truncated,
+				BodyBytes:     mw.totalBytes,
 			}
 			if handlerErr != nil {
 				exchange.Error = handlerErr.Error()
 			}
 
-			if user := auth.GetUser(c); user != nil {
-				exchange.UserID = user.ID
-				exchange.UserName = user.Name
-			}
-
 			mu.Lock()
 			store := traceStore
 			mu.Unlock()
 			select {
 			case logChan <- traceCommand{exchange: &exchange, store: store}:
+				queued = true
 			default:
 				xlog.Warn("Trace channel full, dropping trace")
 			}
@@ -345,6 +359,10 @@ func GetTraces() []APIExchange {
 		return []APIExchange{}
 	}
 	traces := traceBuffer.Values()
+	for _, exchange := range inFlightTraces {
+		exchange.Duration = time.Since(exchange.Timestamp)
+		traces = append(traces, exchange)
+	}
 	mu.Unlock()
 
 	slices.SortFunc(traces, func(a, b APIExchange) int {

core/http/middleware/trace_live_test.go

@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: MIT
+
+package middleware
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/application"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/system"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("live API traces", func() {
+	newApp := func(root string) *application.Application {
+		app, err := application.New(
+			config.EnableTracing,
+			config.WithDataPath(root),
+			config.WithDisableLocalAIAssistant(true),
+			config.WithDisableStats(true),
+			config.WithSystemState(&system.SystemState{
+				Model:   system.Model{ModelsPath: root},
+				Backend: system.Backend{BackendsPath: root},
+			}),
+		)
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(func() { Expect(app.Shutdown()).To(Succeed()) })
+		ClearTraces()
+		return app
+	}
+
+	It("lists a request while its handler is still running", func() {
+		root := GinkgoT().TempDir()
+		app := newApp(root)
+
+		started := make(chan struct{})
+		release := make(chan struct{})
+		DeferCleanup(func() {
+			select {
+			case <-release:
+			default:
+				close(release)
+			}
+		})
+		handler := TraceMiddleware(app)(func(c echo.Context) error {
+			close(started)
+			<-release
+			return c.NoContent(http.StatusNoContent)
+		})
+
+		e := echo.New()
+		req := httptest.NewRequest(http.MethodPost, "/slow", http.NoBody)
+		req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+		rec := httptest.NewRecorder()
+		ctx := e.NewContext(req, rec)
+		ctx.SetPath("/slow")
+		done := make(chan error, 1)
+		go func() {
+			done <- handler(ctx)
+		}()
+		<-started
+
+		var running APIExchange
+		Eventually(func() bool {
+			traces := GetTraces()
+			if len(traces) != 1 {
+				return false
+			}
+			running = traces[0]
+			return running.Request.Path == "/slow"
+		}).Should(BeTrue())
+		Expect(running.Response.Status).To(Equal(0))
+		Expect(running.Duration).To(BeNumerically(">", 0))
+
+		close(release)
+		Expect(<-done).To(Succeed())
+		Eventually(func() []APIExchange { return GetTraces() }).Should(ConsistOf(
+			And(
+				HaveField("ID", running.ID),
+				HaveField("Response.Status", http.StatusNoContent),
+				HaveField("Duration", BeNumerically(">", time.Duration(0))),
+			),
+		))
+	})
+
+	It("removes an in-flight trace when the handler panics", func() {
+		app := newApp(GinkgoT().TempDir())
+		handler := TraceMiddleware(app)(func(echo.Context) error {
+			panic("handler panic")
+		})
+		e := echo.New()
+		req := httptest.NewRequest(http.MethodPost, "/panic", http.NoBody)
+		req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+		ctx := e.NewContext(req, httptest.NewRecorder())
+		ctx.SetPath("/panic")
+
+		func() {
+			defer func() { _ = recover() }()
+			_ = handler(ctx)
+		}()
+
+		Expect(GetTraces()).To(BeEmpty())
+	})
+})

core/http/react-ui/e2e/traces-live.spec.js

@@ -0,0 +1,65 @@
+import { test, expect } from './coverage-fixtures.js'
+
+test('marks an API trace with no response status as in progress', async ({ page }) => {
+  await page.route('**/api/traces?*', route => route.fulfill({
+    json: [{
+      id: 'running-1',
+      timestamp: '2026-08-05T02:00:00Z',
+      duration: 2_000_000_000,
+      request: { method: 'POST', path: '/v1/chat/completions' },
+      response: { status: 0 },
+    }],
+    headers: { 'X-Total-Count': '1' },
+  }))
+  await page.route('**/api/backend-traces?*', route => route.fulfill({ json: [] }))
+
+  await page.goto('/app/traces')
+
+  const row = page.locator('tbody tr').filter({ hasText: '/v1/chat/completions' })
+  await expect(row.getByText('Running', { exact: true })).toBeVisible()
+  await expect(row.locator('[title="In progress"]')).toBeVisible()
+  await expect(row.locator('.fa-check-circle')).toHaveCount(0)
+})
+
+// Regression for #11376: switching from Backend Traces back to API Traces
+// used to crash the page. `traces` holds whichever list was fetched last, so
+// right after `setActiveTab('api')` — before the refetch effect lands — the
+// API table renders the previous tab's backend rows, which carry no
+// `response` envelope. The status column must tolerate that instead of
+// dereferencing `trace.response.status` and tearing down the React tree.
+test('switching from backend to API traces with a response-less row does not crash', async ({ page }) => {
+  const pageErrors = []
+  page.on('pageerror', (e) => pageErrors.push(e.message))
+
+  await page.route('**/api/traces?*', route => route.fulfill({
+    json: [{
+      id: 'api-1',
+      timestamp: '2026-08-05T02:00:00Z',
+      request: { method: 'POST', path: '/v1/chat/completions' },
+      response: { status: 200 },
+    }],
+    headers: { 'X-Total-Count': '1' },
+  }))
+  await page.route('**/api/backend-traces?*', route => route.fulfill({
+    json: [{
+      id: 'backend-1',
+      type: 'llm',
+      timestamp: '2026-08-05T02:00:00Z',
+      model_name: 'mock-model',
+      summary: 'generated a reply',
+    }],
+    headers: { 'X-Total-Count': '1' },
+  }))
+
+  await page.goto('/app/traces')
+  await expect(page.locator('tbody tr').filter({ hasText: '/v1/chat/completions' })).toBeVisible()
+
+  await page.getByRole('button', { name: /Backend Traces/ }).click()
+  await expect(page.locator('tbody tr').filter({ hasText: 'generated a reply' })).toBeVisible()
+
+  await page.getByRole('button', { name: /API Traces/ }).click()
+  // The stale backend row renders in the API table for one frame; the status
+  // column falls back to a neutral placeholder rather than throwing.
+  await expect(page.locator('tbody tr').filter({ hasText: '/v1/chat/completions' })).toBeVisible()
+  expect(pageErrors).toEqual([])
+})

core/http/react-ui/package-lock.json

@@ -21,9 +21,10 @@
         "@fortawesome/fontawesome-free": "^6.7.2",
         "@lezer/highlight": "^1.2.1",
         "@modelcontextprotocol/ext-apps": "^1.2.2",
-        "@modelcontextprotocol/sdk": "^1.25.1",
+        "@modelcontextprotocol/sdk": "^1.30.0",
         "dompurify": "^3.4.12",
         "highlight.js": "^11.11.1",
+        "hono": "4.12.34",
         "i18next": "^26.0.8",
         "i18next-browser-languagedetector": "^8.2.1",
         "i18next-http-backend": "^3.0.6",
@@ -635,12 +636,12 @@
       }
     },
     "node_modules/@hono/node-server": {
-      "version": "1.19.14",
-      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz",
-      "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.0.tgz",
+      "integrity": "sha512-XovyyCCnBzW+zKu+z/zq8hwNs4KOR5rEMAOxo2f40Q5xoOI37IMm6MIg2COOUtUApo0i6850MTBKH2u4QLGIqg==",
       "license": "MIT",
       "engines": {
-        "node": ">=18.14.1"
+        "node": ">=20"
       },
       "peerDependencies": {
         "hono": "^4"
@@ -944,11 +945,12 @@
       }
     },
     "node_modules/@modelcontextprotocol/sdk": {
-      "version": "1.27.1",
-      "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.27.1.tgz",
-      "integrity": "sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA==",
+      "version": "1.30.0",
+      "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz",
+      "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==",
+      "license": "MIT",
       "dependencies": {
-        "@hono/node-server": "^1.19.9",
+        "@hono/node-server": "^1.19.9 || ^2.0.5",
         "ajv": "^8.17.1",
         "ajv-formats": "^3.0.1",
         "content-type": "^1.0.5",
@@ -1718,10 +1720,11 @@
       "dev": true
     },
     "node_modules/brace-expansion": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
-      "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
+      "version": "1.1.18",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz",
+      "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -2876,9 +2879,9 @@
       "dev": true
     },
     "node_modules/fast-uri": {
-      "version": "3.1.4",
-      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz",
-      "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==",
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
+      "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
       "funding": [
         {
           "type": "github",
@@ -3432,9 +3435,9 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.12.31",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.31.tgz",
-      "integrity": "sha512-zJIHFrl6bq3RDd2YusFNCDlM8qUprxKswyi/OPzPyzKDdyBXDqWx8bZlZ7R+saTdSTatUmb3O7K4SspGPaEOQg==",
+      "version": "4.12.34",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.34.tgz",
+      "integrity": "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA==",
       "license": "MIT",
       "engines": {
         "node": ">=16.9.0"
@@ -4193,9 +4196,9 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
     "node_modules/ip-address": {
-      "version": "10.2.0",
-      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
-      "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==",
+      "version": "10.4.0",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz",
+      "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==",
       "license": "MIT",
       "engines": {
         "node": ">= 12"
@@ -4383,16 +4386,16 @@
       }
     },
     "node_modules/istanbul-lib-processinfo/node_modules/brace-expansion": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
       },
       "engines": {
-        "node": "18 || 20 || >=22"
+        "node": "20 || >=22"
       }
     },
     "node_modules/istanbul-lib-processinfo/node_modules/glob": {
@@ -5278,16 +5281,16 @@
       }
     },
     "node_modules/nyc/node_modules/brace-expansion": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
       },
       "engines": {
-        "node": "18 || 20 || >=22"
+        "node": "20 || >=22"
       }
     },
     "node_modules/nyc/node_modules/convert-source-map": {
@@ -5974,10 +5977,11 @@
       }
     },
     "node_modules/quick-temp/node_modules/brace-expansion": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz",
-      "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==",
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
+      "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0"
       }
@@ -6569,16 +6573,16 @@
       }
     },
     "node_modules/spawn-wrap/node_modules/brace-expansion": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
       },
       "engines": {
-        "node": "18 || 20 || >=22"
+        "node": "20 || >=22"
       }
     },
     "node_modules/spawn-wrap/node_modules/foreground-child": {
@@ -6902,16 +6906,16 @@
       }
     },
     "node_modules/test-exclude/node_modules/brace-expansion": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
+      "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
       },
       "engines": {
-        "node": "18 || 20 || >=22"
+        "node": "20 || >=22"
       }
     },
     "node_modules/test-exclude/node_modules/glob": {
@@ -7134,9 +7138,9 @@
       }
     },
     "node_modules/undici": {
-      "version": "7.28.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz",
-      "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==",
+      "version": "7.29.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
+      "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==",
       "dev": true,
       "license": "MIT",
       "engines": {

core/http/react-ui/package.json

@@ -19,7 +19,7 @@
     "coverage:report": "nyc report"
   },
   "overrides": {
-    "hono": "4.12.25"
+    "hono": "4.12.34"
   },
   "dependencies": {
     "@codemirror/autocomplete": "^6.18.6",
@@ -35,10 +35,10 @@
     "@fortawesome/fontawesome-free": "^6.7.2",
     "@lezer/highlight": "^1.2.1",
     "@modelcontextprotocol/ext-apps": "^1.2.2",
-    "@modelcontextprotocol/sdk": "^1.25.1",
+    "@modelcontextprotocol/sdk": "^1.30.0",
     "dompurify": "^3.4.12",
     "highlight.js": "^11.11.1",
-    "hono": "4.12.25",
+    "hono": "4.12.34",
     "i18next": "^26.0.8",
     "i18next-browser-languagedetector": "^8.2.1",
     "i18next-http-backend": "^3.0.6",

core/http/react-ui/src/pages/Traces.jsx

@@ -664,10 +664,18 @@ export default function Traces() {
                     <td><span className="badge badge-info">{trace.request?.method || '-'}</span></td>
                     <td className="text-mono text-sm">{trace.request?.path || '-'}</td>
                     <td className="text-sub cell-clip" title={trace.user_name || trace.user_id || ''}>{trace.user_name || trace.user_id || '-'}</td>
-                    <td><span className={`badge ${(trace.response?.status || 0) < 400 ? 'badge-success' : 'badge-error'}`}>{trace.response?.status || '-'}</span></td>
+                    <td>
+                      {trace.response?.status === 0
+                        ? <span className="badge badge-info">Running</span>
+                        : trace.response?.status == null
+                        ? <span className="badge badge--soft">-</span>
+                        : <span className={`badge ${trace.response.status < 400 ? 'badge-success' : 'badge-error'}`}>{trace.response.status}</span>}
+                    </td>
                     <td><LatencyCell ns={trace.duration} max={slowestTrace} /></td>
                     <td className="text-center">
-                      {trace.error
+                      {trace.response?.status === 0
+                        ? <i className="fas fa-spinner fa-spin text-primary" title="In progress" />
+                        : trace.error
                         ? <i className="fas fa-times-circle text-error" title={trace.error} />
                         : <i className="fas fa-check-circle text-success" />}
                     </td>

core/services/advisorylock/leader_loop_test.go

@@ -54,62 +54,57 @@ var _ = Describe("RunLeaderLoop", func() {
 				close(done)
 			}()
 
-			// Let it run a bit then cancel
-			time.Sleep(150 * time.Millisecond)
+			Eventually(func() int32 {
+				return atomic.LoadInt32(&callCount)
+			}, 500*time.Millisecond, 10*time.Millisecond).Should(BeNumerically(">=", 1))
 			cancel()
 
-			// RunLeaderLoop should return
 			Eventually(done, 500*time.Millisecond).Should(BeClosed())
-
-			// Record count after cancellation
-			countAfterCancel := atomic.LoadInt32(&callCount)
-			time.Sleep(150 * time.Millisecond)
-			countLater := atomic.LoadInt32(&callCount)
-
-			Expect(countLater).To(Equal(countAfterCancel),
-				"function should stop being called after context cancellation")
 		})
 
 		It("only one leader executes at a time (two concurrent loops)", func() {
 			db := testutil.SetupTestDB()
 			const lockKey int64 = 5002
 
-			var (
-				mu         sync.Mutex
-				maxRunning int32
-				running    int32
-			)
+			var running int32
+			entered := make(chan struct{}, 2)
+			release := make(chan struct{})
+			var releaseOnce sync.Once
 
 			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
+			done := make(chan struct{}, 2)
+			DeferCleanup(func() {
+				cancel()
+				releaseOnce.Do(func() { close(release) })
+			})
 
 			fn := func() {
-				cur := atomic.AddInt32(&running, 1)
-				mu.Lock()
-				if cur > maxRunning {
-					maxRunning = cur
+				atomic.AddInt32(&running, 1)
+				select {
+				case entered <- struct{}{}:
+				default:
 				}
-				mu.Unlock()
-
-				time.Sleep(30 * time.Millisecond)
-
+				<-release
 				atomic.AddInt32(&running, -1)
 			}
 
-			// Start two competing leader loops with the same lock key
-			go RunLeaderLoop(ctx, db, lockKey, 50*time.Millisecond, fn)
-			go RunLeaderLoop(ctx, db, lockKey, 50*time.Millisecond, fn)
+			for range 2 {
+				go func() {
+					RunLeaderLoop(ctx, db, lockKey, 1*time.Millisecond, fn)
+					done <- struct{}{}
+				}()
+			}
+
+			Eventually(entered, 500*time.Millisecond).Should(Receive())
+			Consistently(func() int32 {
+				return atomic.LoadInt32(&running)
+			}, 50*time.Millisecond, 5*time.Millisecond).Should(Equal(int32(1)),
+				"expected only the lock holder to run while both loops tick")
 
-			// Let them run for a while
-			time.Sleep(400 * time.Millisecond)
 			cancel()
-
-			mu.Lock()
-			observed := maxRunning
-			mu.Unlock()
-
-			Expect(observed).To(BeNumerically("<=", 1),
-				"expected at most 1 goroutine running the leader function at a time")
+			releaseOnce.Do(func() { close(release) })
+			Eventually(done, 500*time.Millisecond).Should(Receive())
+			Eventually(done, 500*time.Millisecond).Should(Receive())
 		})
 	})
 })

docker-compose.distributed.yaml

@@ -74,6 +74,9 @@ services:
       GODEBUG: "netdns=go"
       # Paths
       MODELS_PATH: /models
+      # Avoid probing remote gallery GGUF metadata during container startup.
+      # Remove this line or set a positive limit to opt back into cache warming.
+      LOCALAI_VRAM_WARM_LIMIT: "0"
     volumes:
       - frontend_models:/models
       - frontend_data:/data

docker-compose.yaml

@@ -18,6 +18,9 @@ services:
       - .env
     environment:
       - MODELS_PATH=/models
+      # Avoid probing remote gallery GGUF metadata during container startup.
+      # Remove this line or set a positive limit to opt back into cache warming.
+      - LOCALAI_VRAM_WARM_LIMIT=0
     #  - DEBUG=true
     ## Agents (LocalAGI) - https://localai.io/features/agents/
     #  - LOCALAI_DISABLE_AGENTS=false

docs/content/advanced/vram-management.md

@@ -477,6 +477,11 @@ then on.
 | `LOCALAI_VRAM_WARM_LIMIT` | `300` | How many gallery entries to warm at startup, estimates and variants alike. Set to `0` to disable the warm-up entirely. |
 | `LOCALAI_VRAM_WARM_CONCURRENCY` | `4` | How many estimates to run at once. |
 
+The provided Docker Compose configurations set `LOCALAI_VRAM_WARM_LIMIT=0`
+as a defensive default, so container startup does not probe remote GGUF files.
+Remove that override or set it to a positive number to opt into background
+warming.
+
 ```bash
 # Air-gapped, or you would rather not make the requests at all
 LOCALAI_VRAM_WARM_LIMIT=0 local-ai run

docs/content/blog/2024-04-24-reranker-api.md

@@ -1,10 +0,0 @@
-+++
-title = "Reranker API"
-date = 2024-04-24
-description = "A new reranker backend implementing the Jina rerankers API."
-url = "/blog/reranker-api/"
-+++
-
-A new reranker backend lands, implementing the Jina rerankers API, in [PR #2121](https://github.com/mudler/LocalAI/pull/2121).
-
-See [Reranker]({{% relref "features/reranker" %}}).

docs/content/blog/2024-05-14-distributed-and-p2p-inferencing.md

@@ -1,13 +0,0 @@
-+++
-title = "Distributed and decentralized P2P inferencing"
-date = 2024-05-14
-description = "Distributed llama.cpp inferencing, followed by fully decentralized peer-to-peer inference."
-url = "/blog/distributed-and-p2p-inferencing/"
-+++
-
-Two changes that set up everything LocalAI later built on top of:
-
-- [Distributed llama.cpp inferencing](https://github.com/mudler/LocalAI/pull/2324), splitting a model across machines.
-- [Totally decentralized, private, distributed peer-to-peer inference](https://github.com/mudler/LocalAI/pull/2343).
-
-See [Distributed inferencing]({{% relref "features/distributed_inferencing" %}}).

docs/content/blog/2024-08-02-p2p-federation-and-swarms.md

@@ -1,15 +0,0 @@
-+++
-title = "P2P dashboard, federated mode and AI swarms"
-date = 2024-08-02
-description = "A P2P dashboard, federation, AI swarms, global community pools, FLUX-1 support and the P2P Explorer."
-url = "/blog/p2p-federation-and-swarms/"
-+++
-
-The peer-to-peer work matured over July and August:
-
-- [A P2P dashboard, federated mode and AI swarms](https://github.com/mudler/LocalAI/pull/2723).
-- [Global community pools](https://github.com/mudler/LocalAI/issues/3113), for sharing federated instances and workers.
-- FLUX-1 support.
-- The [P2P Explorer](https://explorer.localai.io).
-
-See [Distributed inferencing]({{% relref "features/distributed_inferencing" %}}).

docs/content/blog/2024-10-01-examples-moved-out.md

@@ -1,8 +0,0 @@
-+++
-title = "Examples move to LocalAI-examples"
-date = 2024-10-01
-description = "The examples directory leaves the main repository and gets its own home."
-url = "/blog/examples-moved-out/"
-+++
-
-The examples have moved out of the main repository into [LocalAI-examples](https://github.com/mudler/LocalAI-examples), where they can be versioned and maintained independently of the runtime.

docs/content/blog/2024-11-20-vad-and-bark-cpp.md

@@ -1,9 +0,0 @@
-+++
-title = "Voice Activity Detection and bark.cpp"
-date = 2024-11-20
-description = "Silero-based Voice Activity Detection, plus a bark.cpp backend for audio generation."
-url = "/blog/vad-and-bark-cpp/"
-+++
-
-- [Voice Activity Detection](https://github.com/mudler/LocalAI/pull/4204), via a Silero VAD backend. See [Voice activity detection]({{% relref "features/voice-activity-detection" %}}).
-- [A bark.cpp backend](https://github.com/mudler/LocalAI/pull/4287) for audio generation.

docs/content/blog/2024-12-03-stablediffusion-cpp-backend.md

@@ -1,10 +0,0 @@
-+++
-title = "stablediffusion.cpp backend (ggml)"
-date = 2024-12-03
-description = "A ggml-based stablediffusion.cpp backend for image generation."
-url = "/blog/stablediffusion-cpp-backend/"
-+++
-
-A ggml-based `stablediffusion.cpp` backend lands for image generation, in [PR #4289](https://github.com/mudler/LocalAI/pull/4289).
-
-See [Image generation]({{% relref "features/image-generation" %}}).

docs/content/blog/2025-07-24-modular-backend-architecture.md

@@ -1,12 +0,0 @@
-+++
-title = "Backends move outside the main binary"
-date = 2025-07-24
-description = "All backends migrate out of the main binary, leaving a lightweight modular core that pulls engines on demand."
-url = "/blog/modular-backend-architecture/"
-+++
-
-All backends have been migrated outside the main binary. The core stays small, and each backend is an isolated service installed on demand.
-
-This is the architecture LocalAI still runs on: install, update or remove engines independently, and mix CPU, NVIDIA, AMD, Intel, Apple Silicon, Vulkan and Jetson in one deployment.
-
-See [Backends]({{% relref "features/backends" %}}) and the [v3.2.0 release notes](https://github.com/mudler/LocalAI/releases/tag/v3.2.0).

docs/content/blog/2025-08-12-apple-silicon-backends.md

@@ -1,10 +0,0 @@
-+++
-title = "MLX, MLX-VLM, Diffusers and llama.cpp on Apple Silicon"
-date = 2025-08-12
-description = "Apple Silicon gains first-class backend coverage."
-url = "/blog/apple-silicon-backends/"
-+++
-
-MLX, MLX-VLM, Diffusers and llama.cpp are now supported on Apple Silicon, giving Mac users the same backend choice available elsewhere.
-
-Released as part of [v3.4.0](https://github.com/mudler/LocalAI/releases/tag/v3.4.0).

docs/content/blog/2025-09-03-launcher-and-extended-backends.md

@@ -1,13 +0,0 @@
-+++
-title = "New launcher, extended backend support, MLX-Audio and WAN 2.2"
-date = 2025-09-03
-description = "A desktop launcher for macOS and Linux, wider backend coverage for Mac and Nvidia L4T, MLX-Audio and WAN 2.2."
-url = "/blog/launcher-and-extended-backends/"
-+++
-
-- A new [launcher app](https://github.com/mudler/LocalAI/pull/6127) for macOS and Linux, so LocalAI can be started and managed without the terminal.
-- Extended backend support for Mac and Nvidia L4T.
-- MLX-Audio.
-- WAN 2.2.
-
-Released as part of [v3.5.0](https://github.com/mudler/LocalAI/releases/tag/v3.5.0).

docs/content/blog/2025-10-05-mcp-support.md

@@ -1,10 +0,0 @@
-+++
-title = "Model Context Protocol (MCP) support"
-date = 2025-10-05
-description = "Agentic capabilities through MCP, with a new chat/completion endpoint that can call MCP tools."
-url = "/blog/mcp-support/"
-+++
-
-LocalAI gains [Model Context Protocol](https://modelcontextprotocol.io) support for agentic capabilities, through [a new chat/completion endpoint](https://github.com/mudler/LocalAI/pull/6381) that can reach MCP tools, plus [a UI toggle to enable it](https://github.com/mudler/LocalAI/pull/6400).
-
-See [MCP]({{% relref "features/mcp" %}}).

docs/content/blog/2025-11-24-import-models-via-url-and-chat-history.md

@@ -1,11 +0,0 @@
-+++
-title = "Import models via URL, multiple chats and history"
-date = 2025-11-24
-description = "Point LocalAI at a model URL to import it, and keep several chat threads with their history in the UI."
-url = "/blog/import-models-via-url-and-chat-history/"
-+++
-
-Two usability changes:
-
-- [Import models via URL](https://github.com/mudler/LocalAI/pull/7245). Paste a model URL and LocalAI handles the download and configuration.
-- [Multiple chats and history](https://github.com/mudler/LocalAI/pull/7325) in the UI, so conversations persist and can run in parallel.

docs/content/blog/2025-12-16-memory-reclaimer-and-multi-gpu-fitting.md

@@ -1,12 +0,0 @@
-+++
-title = "Dynamic memory reclaimer, multi-GPU fitting and Vibevoice"
-date = 2025-12-16
-description = "Reclaim GPU memory from idle models, fit llama.cpp models across multiple GPUs automatically, and generate long-form speech with Vibevoice."
-url = "/blog/memory-reclaimer-and-multi-gpu-fitting/"
-+++
-
-Three additions this month:
-
-- [A dynamic memory resource reclaimer](https://github.com/mudler/LocalAI/pull/7583), which frees GPU memory held by idle models.
-- [Automatic multi-GPU model fitting for llama.cpp](https://github.com/mudler/LocalAI/pull/7584), so a model too large for one device is split across several without hand-tuning.
-- [The Vibevoice backend](https://github.com/mudler/LocalAI/pull/7494) for long-form speech.

docs/content/blog/2026-01-18-localai-3-10-0.md

@@ -1,17 +0,0 @@
-+++
-title = "LocalAI 3.10.0"
-date = 2026-01-18
-description = "Anthropic API support, the Open Responses API, video and image generation with LTX-2, unified GPU backends, tool streaming, Moonshine and Pocket-TTS."
-url = "/blog/localai-3-10-0/"
-+++
-
-LocalAI 3.10.0 is out.
-
-- Anthropic API support.
-- The Open Responses API.
-- Video and image generation with LTX-2.
-- Unified GPU backends.
-- Tool streaming.
-- Moonshine and Pocket-TTS.
-
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v3.10.0).

docs/content/blog/2026-02-05-realtime-api-and-ace-step.md

@@ -1,11 +0,0 @@
-+++
-title = "Realtime API and ACE-Step 1.5"
-date = 2026-02-05
-description = "Audio-to-audio with tool calling through the Realtime API, plus ACE-Step 1.5 music generation."
-url = "/blog/realtime-api-and-ace-step/"
-+++
-
-Two additions this month:
-
-- [The Realtime API for audio-to-audio with tool calling](https://github.com/mudler/LocalAI/pull/6245). See [Realtime API]({{% relref "features/openai-realtime" %}}).
-- [ACE-Step 1.5 support](https://github.com/mudler/LocalAI/pull/8396) for music generation.

docs/content/blog/2026-03-14-localai-4-0-0.md

@@ -1,16 +0,0 @@
-+++
-title = "LocalAI 4.0.0: native agentic orchestration"
-date = 2026-03-14
-description = "The Agenthub community hub, a full React UI rewrite with Canvas mode, MCP Apps with tool streaming, WebRTC realtime audio, and MLX-distributed."
-url = "/blog/localai-4-0-0/"
-+++
-
-LocalAI 4.0.0 brings agentic orchestration into the core.
-
-- Native agentic orchestration, with the new [Agenthub](https://agenthub.localai.io) community hub.
-- A full React UI rewrite, including Canvas mode.
-- [MCP Apps and client-side MCP](https://github.com/mudler/LocalAI/pull/8947) with tool streaming.
-- [WebRTC realtime audio](https://github.com/mudler/LocalAI/pull/8790).
-- [MLX-distributed](https://github.com/mudler/LocalAI/pull/8801).
-
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.0.0).

docs/content/blog/2026-04-02-localai-4-1-0.md

@@ -1,17 +0,0 @@
-+++
-title = "LocalAI 4.1.0: LocalAI becomes a control tower"
-date = 2026-04-02
-description = "Distributed cluster mode with VRAM-aware routing and autoscaling, a multi-user platform with OIDC, per-user quotas, in-UI fine-tuning, and a visual pipeline editor."
-url = "/blog/localai-4-1-0/"
-+++
-
-LocalAI 4.1.0 turns LocalAI into a control tower rather than a single inference server.
-
-- Distributed cluster mode, with VRAM-aware smart routing and autoscaling.
-- A multi-user platform with OIDC and API keys.
-- Per-user quotas with predictive analytics.
-- In-UI fine-tuning with TRL, including automatic export to GGUF.
-- An on-the-fly quantization backend.
-- A visual pipeline editor.
-
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.1.0).

docs/content/blog/2026-04-22-face-recognition-backend.md

@@ -1,20 +0,0 @@
-+++
-title = "Face recognition backend"
-date = 2026-04-22
-description = "insightface-powered 1:1 verification, 1:N identification, face embedding, detection and demographic analysis."
-url = "/blog/face-recognition-backend/"
-+++
-
-A new face recognition backend, powered by `insightface`, covering:
-
-- 1:1 verification
-- 1:N identification
-- Face embedding
-- Face detection
-- Demographic analysis
-
-It ships with two model options: the non-commercial `buffalo_l`, and an Apache 2.0 alternative from the OpenCV Zoo.
-
-See [Face recognition]({{% relref "features/face-recognition" %}}). Shipped in [PR #9480](https://github.com/mudler/LocalAI/pull/9480).
-
-The engine was later rewritten from scratch in C++/ggml: see [Native biometric backends]({{% relref "blog/2026-06-28-native-biometric-backends" %}}).

docs/content/blog/2026-05-04-audio-transform.md

@@ -1,19 +0,0 @@
-+++
-title = "Audio Transform"
-date = 2026-05-04
-description = "A generic audio-in / audio-out endpoint with an optional reference signal. First implementation: LocalVQE, a joint AEC, noise suppression and dereverberation engine."
-url = "/blog/audio-transform/"
-+++
-
-Audio Transform is a generic audio-in / audio-out endpoint, with an optional reference signal for tasks that need one.
-
-The first implementation is [LocalVQE](https://github.com/localai-org/LocalVQE), a C++ backend doing joint acoustic echo cancellation, noise suppression and dereverberation in a DeepVQE-style model.
-
-Both call styles are supported:
-
-- Batch, via `POST /audio/transformations`.
-- Bidirectional streaming, via the `/audio/transformations/stream` WebSocket.
-
-Studio gains a "Transform" tab with synchronized waveform players for the input, reference and output signals.
-
-See [Audio transform]({{% relref "features/audio-transform" %}}). Shipped in [PR #9640](https://github.com/mudler/LocalAI/pull/9640).

docs/content/blog/2026-05-05-speaker-diarization.md

@@ -1,17 +0,0 @@
-+++
-title = "Speaker diarization"
-date = 2026-05-05
-description = "A /v1/audio/diarization endpoint returning who spoke when, backed by sherpa-onnx and vibevoice-cpp."
-url = "/blog/speaker-diarization/"
-+++
-
-`POST /v1/audio/diarization` is a new endpoint that returns "who spoke when" as a list of segments.
-
-Two backends serve it:
-
-- `sherpa-onnx` for pure diarization, combining pyannote-3.0, speaker embeddings and clustering.
-- `vibevoice-cpp` for diarization bundled with long-form ASR.
-
-Responses are available as `json`, `verbose_json` or `rttm`.
-
-See [Audio diarization]({{% relref "features/audio-diarization" %}}). Shipped in [PR #9654](https://github.com/mudler/LocalAI/pull/9654).

docs/content/blog/2026-05-24-localai-4-3-0.md

@@ -1,15 +0,0 @@
-+++
-title = "LocalAI 4.3.0"
-date = 2026-05-24
-description = "llama.cpp prompt cache on by default, keyless cosign signing of backend images, per-key and per-user usage attribution, and Distributed v3."
-url = "/blog/localai-4-3-0/"
-+++
-
-LocalAI 4.3.0 is out.
-
-- [Prompt cache on by default for llama.cpp](https://github.com/mudler/LocalAI/pull/9925). Repeated system prompts collapse from minutes to seconds.
-- [Keyless cosign signing of backend OCI images](https://github.com/mudler/LocalAI/pull/9823).
-- [Per-API-key and per-user usage attribution](https://github.com/mudler/LocalAI/pull/9920).
-- Distributed v3, with [per-request replica routing](https://github.com/mudler/LocalAI/pull/9968).
-
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.3.0).

docs/content/blog/2026-06-11-realtime-voice-assistant-demo.md

@@ -1,12 +0,0 @@
-+++
-title = "Realtime voice assistant demo and pipeline streaming"
-date = 2026-06-11
-description = "A tiny Go client for the Realtime API with a full talk-back loop and tool calling, plus streaming of the realtime pipeline stages."
-url = "/blog/realtime-voice-assistant-demo/"
-+++
-
-The new [realtime voice assistant demo](https://github.com/localai-org/localai-realtime-demo) is a small Go client for the Realtime API with a complete talk-back voice loop and tool calling. It is intended as a reference you can read end to end.
-
-On the server side, two supporting changes landed: [streaming of the realtime LLM, TTS and transcription pipeline stages](https://github.com/mudler/LocalAI/pull/10176), and [configurable WebRTC ICE candidates](https://github.com/mudler/LocalAI/pull/10231).
-
-See [Realtime API]({{% relref "features/openai-realtime" %}}).

docs/content/blog/2026-06-12-distributed-mode-hardening.md

@@ -1,16 +0,0 @@
-+++
-title = "Distributed mode hardening"
-date = 2026-06-12
-description = "Prefix-cache-aware routing, a production-ready request router, ds4 layer-split inference, NATS JWT auth with TLS/mTLS, and resumable uploads."
-url = "/blog/distributed-mode-hardening/"
-+++
-
-Distributed mode picked up a round of production hardening:
-
-- [Prefix-cache-aware routing](https://github.com/mudler/LocalAI/pull/10071), so requests sharing a prompt prefix land on the replica that already holds it.
-- [A production-ready request router with auto-sized embedding and rerank batches](https://github.com/mudler/LocalAI/pull/10104).
-- [ds4 layer-split distributed inference](https://github.com/mudler/LocalAI/pull/10098).
-- [NATS JWT auth plus TLS/mTLS](https://github.com/mudler/LocalAI/pull/10159).
-- [Resumable file uploads](https://github.com/mudler/LocalAI/pull/10109).
-
-See [Distributed inferencing]({{% relref "features/distributed_inferencing" %}}).

docs/content/blog/2026-06-12-new-backends-and-models.md

@@ -1,15 +0,0 @@
-+++
-title = "New backends and models: locate-anything.cpp, Ideogram4, Gemma 4"
-date = 2026-06-12
-description = "Open-vocabulary object detection via ggml, Ideogram4 image generation, llama.cpp video input, and the Gemma 4 QAT family with MTP pairs."
-url = "/blog/new-backends-and-models-june-2026/"
-+++
-
-A batch of new capability this month:
-
-- [locate-anything.cpp](https://github.com/mudler/LocalAI/pull/10264) for open-vocabulary object detection via ggml.
-- [Ideogram4 image generation](https://github.com/mudler/LocalAI/pull/10201) in `stablediffusion-ggml`.
-- [llama.cpp video input](https://github.com/mudler/LocalAI/pull/10216).
-- [The Gemma 4 QAT family with MTP speculative-decoding pairs](https://github.com/mudler/LocalAI/pull/10215).
-
-Plus two usability additions: an [interactive CLI chat mode](https://github.com/mudler/LocalAI/pull/10226) and [RAG source citations in agent responses](https://github.com/mudler/LocalAI/pull/10228).

docs/content/blog/2026-06-13-speech-push-parakeet-crispasr-piper.md

@@ -1,17 +0,0 @@
-+++
-title = "A big speech push: parakeet.cpp, CrispASR and 60 Piper voices"
-date = 2026-06-13
-description = "Segment timestamps, multilingual streaming, dynamic batching and CUDA graphs for parakeet.cpp, plus a new ASR/TTS backend and a large Piper voice drop."
-url = "/blog/speech-push-parakeet-crispasr-piper/"
-+++
-
-A concentrated round of speech work landed this month.
-
-[parakeet.cpp](https://github.com/mudler/parakeet.cpp), our ASR engine, gained:
-
-- [NeMo-faithful segment timestamps](https://github.com/mudler/LocalAI/pull/10207)
-- [a multilingual streaming Nemotron-3.5 model](https://github.com/mudler/LocalAI/pull/10199)
-- [dynamic batching for concurrent transcription](https://github.com/mudler/LocalAI/pull/10112)
-- [CUDA graphs](https://github.com/mudler/LocalAI/pull/10273)
-
-Alongside it, the new [CrispASR backend](https://github.com/mudler/LocalAI/pull/10099) adds multi-architecture ASR and TTS, and [60 Piper TTS voices across 42 languages](https://github.com/mudler/LocalAI/pull/10296) land in the gallery, together with [per-request TTS instructions and parameters](https://github.com/mudler/LocalAI/pull/10172).

docs/content/blog/2026-06-18-pii-analyze-redact-api.md

@@ -1,15 +0,0 @@
-+++
-title = "PII analyze and redact API"
-date = 2026-06-18
-description = "The PII detection pipeline becomes a standalone service, callable without routing a chat request through the middleware."
-url = "/blog/pii-analyze-redact-api/"
-+++
-
-The PII detection pipeline (NER plus restricted-regex pattern tiers) is now reachable directly, without routing a chat request through the middleware:
-
-- `POST /api/pii/analyze` returns the detected entity spans.
-- `POST /api/pii/redact` returns the sanitised text, or `400 pii_blocked`.
-
-Events also gain an `origin` field (`middleware`, `proxy`, `pii_analyze`, `pii_redact`), so `/api/pii/events` can be filtered by which surface produced them.
-
-See [Middleware]({{% relref "operations/middleware" %}}#analyze--redact-api). Shipped in [PR #10360](https://github.com/mudler/LocalAI/pull/10360).

docs/content/blog/2026-06-22-sound-classification.md

@@ -1,12 +0,0 @@
-+++
-title = "Sound classification with ced.cpp"
-date = 2026-06-22
-description = "A new /v1/audio/classification endpoint for audio tagging, returning scored AudioSet labels."
-url = "/blog/sound-classification/"
-+++
-
-`POST /v1/audio/classification` is a new endpoint for audio tagging and sound-event classification. It returns scored [AudioSet](https://research.google.com/audioset/) labels: baby cry, glass breaking, alarms, and several hundred others.
-
-It is backed by [ced.cpp](https://github.com/localai-org/ced.cpp), a 527-class AudioSet tagger ported to ggml by the LocalAI team.
-
-See [Audio classification]({{% relref "features/audio-classification" %}}). Shipped in [PR #10425](https://github.com/mudler/LocalAI/pull/10425).

docs/content/blog/2026-06-28-native-biometric-backends.md

@@ -1,15 +0,0 @@
-+++
-title = "Native biometric backends: voice-detect.cpp and face-detect.cpp"
-date = 2026-06-28
-description = "Two from-scratch C++/ggml engines replace the heavier Python insightface and speaker-recognition backends."
-url = "/blog/native-biometric-backends/"
-+++
-
-Two new biometric engines built by the LocalAI team, both from-scratch C++/ggml implementations with no Python and no onnxruntime at inference time:
-
-- [voice-detect.cpp](https://github.com/localai-org/voice-detect.cpp) for speaker recognition and voice analysis: ECAPA-TDNN, WeSpeaker, ERes2Net, CAM++, and wav2vec2 age/gender/emotion.
-- [face-detect.cpp](https://github.com/mudler/face-detect.cpp) for face detection, recognition, demographics and anti-spoofing: SCRFD/ArcFace and YuNet/SFace.
-
-Both ship self-contained GGUF weights, hold bit-exact parity with the reference implementations, and reach cuDNN parity on GPU. They replace the heavier Python `insightface` and `speaker-recognition` backends.
-
-Shipped in [PR #10441](https://github.com/mudler/LocalAI/pull/10441).

docs/content/blog/2026-06-30-concurrent-scoring-and-pii-ner.md

@@ -1,15 +0,0 @@
-+++
-title = "Concurrent scoring and PII NER on llama.cpp"
-date = 2026-06-30
-description = "Score and TokenClassify now ride llama.cpp's server task queue instead of locking the context, so they run alongside chat traffic."
-url = "/blog/concurrent-scoring-and-pii-ner/"
-+++
-
-The `Score` primitive (used by the router classifier) and `TokenClassify` (used by the PII NER tier) previously locked the llama.cpp context for the duration of the call. They now ride llama.cpp's server task queue instead.
-
-What changes as a result:
-
-- Scoring and token classification run concurrently with chat, completion and embedding traffic, and with each other.
-- The `known_usecases` restriction that forced dedicated scorer and NER model configs on `llama-cpp` is lifted.
-- Repeated scoring calls reuse the prompt KV cache across candidates.
-- Scoring inputs are no longer capped by the physical batch size.

docs/content/blog/2026-07-05-model-capabilities-endpoint.md

@@ -1,14 +0,0 @@
-+++
-title = "Model capabilities endpoint"
-date = 2026-07-05
-description = "GET /v1/models/capabilities reports what each model can do and which modalities it accepts, so clients stop guessing from backend names."
-url = "/blog/model-capabilities-endpoint/"
-+++
-
-`GET /v1/models/capabilities` is a new endpoint: an additive superset of `/v1/models` that reports each model's `capabilities` alongside its `input_modalities` and `output_modalities` (`text`, `image`, `audio`, `video`).
-
-The practical effect is that a client can decide where to send an attachment by asking the server, instead of pattern-matching on backend names. Modalities are either inferred by LocalAI or declared explicitly in the model config.
-
-Because the endpoint is additive, existing `/v1/models` consumers are unaffected.
-
-See [API discovery]({{% relref "features/api-discovery" %}}#model-capabilities). Shipped in [PR #10687](https://github.com/mudler/LocalAI/pull/10687).

docs/content/blog/2026-07-12-longcat-video-and-avatar-generation.md

@@ -1,16 +0,0 @@
-+++
-title = "LongCat video and avatar generation"
-date = 2026-07-12
-description = "A dedicated CUDA backend for LongCat-Video text/image-to-video and LongCat-Video-Avatar-1.5 speech-driven avatars."
-url = "/blog/longcat-video-and-avatar-generation/"
-+++
-
-LocalAI gains a dedicated CUDA backend for the LongCat family: `LongCat-Video` for text-to-video and image-to-video, and `LongCat-Video-Avatar-1.5` for speech-driven avatars.
-
-Highlights:
-
-- Multi-segment continuation, so a clip can be extended beyond a single generation window.
-- Portrait and recorded-audio inputs wired into Studio.
-- An SDPA CUDA 13 ARM64 build, which makes the backend usable on DGX Spark.
-
-See [Video generation]({{% relref "features/video-generation" %}}) for configuration and the available model entries. Shipped in [PR #10792](https://github.com/mudler/LocalAI/pull/10792).

docs/content/blog/_index.md

@@ -1,17 +0,0 @@
-+++
-title = "Blog"
-weight = 10
-icon = "newspaper"
-alwaysopen = false
-aliases = ["/basics/news/", "/whats-new/"]
-+++
-
-Announcements, release write-ups and feature notes from the LocalAI team.
-
-Full changelogs for every version live on [GitHub Releases](https://github.com/mudler/LocalAI/releases). This page is the narrative archive: what shipped, and why it matters.
-
-{{% notice tip %}}
-Prefer a feed reader? Subscribe to [/blog/index.xml](/blog/index.xml).
-{{% /notice %}}
-
-{{< postlist >}}

docs/content/features/backends.md

@@ -72,6 +72,44 @@ tags:
   - "text-generation"
 ```
 
+### Verifying OCI Backends
+
+Backend galleries can require keyless Sigstore signatures for every OCI image
+they provide. Add a `verification` policy to the gallery configuration, then
+enable strict integrity mode:
+
+```bash
+export LOCALAI_BACKEND_GALLERIES='[{"name":"localai","url":"github:mudler/LocalAI/backend/index.yaml@master","verification":{"issuer":"https://token.actions.githubusercontent.com","identity_regex":"^https://github\\.com/mudler/LocalAI/\\.github/workflows/backend_merge\\.yml@refs/(heads/master|tags/.+)$"}}]'
+export LOCALAI_REQUIRE_BACKEND_INTEGRITY=1
+local-ai run
+```
+
+The policy pins the Fulcio issuer and the GitHub Actions workflow identity that
+signed the image. The identity expression covers development images produced
+from `master` and release images produced from tags. Use a narrower expression
+if your deployment only accepts one release channel.
+
+Without strict mode, an OCI gallery without a verification policy installs
+with a warning. With strict mode, LocalAI refuses galleries without a policy,
+images without a compatible Sigstore bundle, and signatures that do not match
+the configured identity. Existing images published before bundle signing was
+enabled must be rebuilt or re-signed before strict deployments can install
+them.
+
+An optional `not_before` RFC3339 value revokes signatures logged before that
+time. Advance it after a signing-workflow compromise, then rebuild or re-sign
+the trusted images:
+
+```json
+{
+  "verification": {
+    "issuer": "https://token.actions.githubusercontent.com",
+    "identity_regex": "^https://github\\.com/mudler/LocalAI/\\.github/workflows/backend_merge\\.yml@refs/(heads/master|tags/.+)$",
+    "not_before": "2026-08-05T00:00:00Z"
+  }
+}
+```
+
 ## Pre-installing Backends
 
 You can pre-install backends when starting LocalAI using the `LOCALAI_EXTERNAL_BACKENDS` environment variable:

docs/content/features/tracing.md

@@ -9,6 +9,11 @@ LocalAI can retain recent API exchanges and backend operations for inspection
 on the **Traces** page in the management interface. Enable tracing in runtime
 settings or with the existing tracing configuration.
 
+API requests appear while they are still running. Their elapsed duration
+updates when the page refreshes, and the result column marks them as in
+progress until the response completes. In-flight requests live only in memory;
+the completed exchange is what LocalAI adds to the bounded, persistent history.
+
 API and backend trace histories are persisted in separate directories below
 the configured data path. They are restored after a clean service restart,
 whether or not authentication is enabled.

docs/data/version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v4.7.1"
+  "version": "v4.8.0"
 }

gallery/index.yaml

@@ -1,4 +1,101 @@
 ---
+- &qwen3-5-9b-defiant-fable
+  name: "qwen3.5-9b-defiant-fable-mtp"
+  variants:
+    - model: qwen3.5-9b-defiant-fable
+  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+  urls:
+    - https://huggingface.co/DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF
+  description: |
+    Qwen3.5 9B Defiant Fable is an Apache-2.0 multimodal fine-tune for
+    reasoning, coding, creative writing, and roleplay. It retains the 256K
+    context window and vision support of Qwen3.5 while reducing refusals.
+    This default entry uses the NEO-imatrix Q4_K_M build with multi-token
+    prediction enabled for faster generation.
+  license: apache-2.0
+  icon: https://huggingface.co/DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/resolve/main/defiant-fable-9b.png
+  tags:
+    - llm
+    - gguf
+    - cpu
+    - gpu
+    - qwen3.5
+    - reasoning
+    - coding
+    - creative-writing
+    - uncensored
+    - vision
+    - multimodal
+    - mtp
+  last_checked: "2026-08-04"
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+      - vision
+    mmproj: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf
+    options:
+      - use_jinja:true
+      - spec_type:draft-mtp
+      - spec_n_max:6
+      - spec_p_min:0.75
+    parameters:
+      model: llama-cpp/models/qwen3.5-9b-defiant-fable/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/qwen3.5-9b-defiant-fable/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf
+      uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf
+      sha256: d7eb4fac9389d53fa576f64a6ff53e914a00bc7705dc354d1065887565147320
+    - filename: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf
+      uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/mmproj-BF16.gguf
+      sha256: 853698ce7aa6c7ba732478bad280240969ddf7b0fcbf93900046f63903a83383
+- !!merge <<: *qwen3-5-9b-defiant-fable
+  name: "qwen3.5-9b-defiant-fable"
+  variants: []
+  description: |
+    Qwen3.5 9B Defiant Fable in the plain NEO-imatrix Q4_K_M GGUF format.
+    This fallback offers the same multimodal reasoning, coding, and creative
+    capabilities without enabling multi-token prediction.
+  tags:
+    - llm
+    - gguf
+    - cpu
+    - gpu
+    - qwen3.5
+    - reasoning
+    - coding
+    - creative-writing
+    - uncensored
+    - vision
+    - multimodal
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+      - vision
+    mmproj: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/qwen3.5-9b-defiant-fable/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_M.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/qwen3.5-9b-defiant-fable/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_M.gguf
+      uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_M.gguf
+      sha256: d33db5e583b9c9251402e876443791bc979f12af934bfb0630eadfb456279f84
+    - filename: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf
+      uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/mmproj-BF16.gguf
+      sha256: 853698ce7aa6c7ba732478bad280240969ddf7b0fcbf93900046f63903a83383
 - &nemotron-3-embed-1b
   name: "nemotron-3-embed-1b-q4"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
@@ -189,7 +286,7 @@
   files:
     - filename: DeepSeek-V4-Flash-0731-MXFP4.gguf
       uri: huggingface://ggml-org/DeepSeek-V4-Flash-0731-GGUF/DeepSeek-V4-Flash-0731-MXFP4.gguf
-      sha256: c8b46876c3939a6e141f9e4d4aa422981df4a9b84f19e9bb4e1c9a28be31e484
+      sha256: 65f73494afaf27d3add0751a5b716dd2d3e012c66ae0dbbcc1bf8477f92b3ab7
 - name: instella-moe-16b-a3b-think
   url: github:mudler/LocalAI/gallery/virtual.yaml@master
   urls:
@@ -311,7 +408,7 @@
   files:
     - filename: llama-cpp/models/Parable-Granite-4.1-3B-Claude-Fable-5-Q4_K_M/Parable-Granite-4.1-3B-Claude-Fable-5-GGUF-Q4_K_M.gguf
       uri: https://huggingface.co/AnkitAI/Parable-Granite-4.1-3B-Claude-Fable-5-GGUF/resolve/main/Parable-Granite-4.1-3B-Claude-Fable-5-GGUF-Q4_K_M.gguf
-      sha256: 67dc7695d92939c713165761f115c9d892fdff74fcbd987c8bb453b9b8ab645d
+      sha256: dbf202638af23e72508d8316577655d24ba2037fda51ce802b8996977e290bce
 - name: "parable-qwen3-4b-claude-fable-5"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -345,7 +442,7 @@
   files:
     - filename: llama-cpp/models/Parable-Qwen3-4B-Claude-Fable-5-Q4_K_M/Parable-Qwen3-4B-Claude-Fable-5-GGUF-Q4_K_M.gguf
       uri: https://huggingface.co/AnkitAI/Parable-Qwen3-4B-Claude-Fable-5-GGUF/resolve/main/Parable-Qwen3-4B-Claude-Fable-5-GGUF-Q4_K_M.gguf
-      sha256: c94b06a912aa901f3da5689754577ad534415efafc50dcee3f389594a153bf38
+      sha256: 65cc4824fb78ecaf55afdfcdb6dd2e27e1aa805d289db89eae94d32d450403f0
 - name: "parable-granite-4.1-8b-claude-fable-5"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -381,7 +478,7 @@
   files:
     - filename: llama-cpp/models/Parable-Granite-4.1-8B-Claude-Fable-5-Q4_K_M/Parable-Granite-4.1-8B-Claude-Fable-5-GGUF-Q4_K_M.gguf
       uri: https://huggingface.co/AnkitAI/Parable-Granite-4.1-8B-Claude-Fable-5-GGUF/resolve/main/Parable-Granite-4.1-8B-Claude-Fable-5-GGUF-Q4_K_M.gguf
-      sha256: 61a8133c344a0d0a00188395afe33c803e3b973cb4bbfd5ef1fa7110e80bc1c3
+      sha256: 57e464ae3d35253d4351639757dc35e71bab8324d12d49a5870695ce73dc19cf
 - name: "parable-qwen3-8b-claude-fable-5"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -415,7 +512,7 @@
   files:
     - filename: llama-cpp/models/Parable-Qwen3-8B-Claude-Fable-5-Q4_K_M/Parable-Qwen3-8B-Claude-Fable-5-GGUF-Q4_K_M.gguf
       uri: https://huggingface.co/AnkitAI/Parable-Qwen3-8B-Claude-Fable-5-GGUF/resolve/main/Parable-Qwen3-8B-Claude-Fable-5-GGUF-Q4_K_M.gguf
-      sha256: 956070afc8023b8665fe450842f7be76b505b53d142460fd9b588222f4e16112
+      sha256: 4532d2379d38a37279866a030e51d419561f9d4d22fee00d2a33647d66f05065
 - &pocket-35b
   name: "pocket-35b"
   variants:
@@ -785,35 +882,18 @@
 - name: "qwen3.6-35b-a3b-uncensored-genesis-hermes-v6"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
+    - https://huggingface.co/HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive
     - https://huggingface.co/LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V6-GGUF
   description: |
-    # Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive
+    Qwen3.6-35B-A3B Uncensored Genesis Hermes V6 is LuffyTheFox's multimodal,
+    agentic derivative of HauhauCS's uncensored Qwen3.6-35B-A3B model. It
+    combines Genesis tensor calibration with Hermes function-calling data while
+    retaining the 35B mixture-of-experts architecture, roughly 3B active
+    parameters per token, and the native 262K-token context window.
 
-    > **Join the Discord** for updates, roadmaps, projects, or just to chat.
-
-    Qwen3.6-35B-A3B uncensored by HauhauCS. **0/465 Refusals.**
-
-    > **HuggingFace's "Hardware Compatibility" widget doesn't recognize K_P quants** — it may show fewer files than actually exist. Click **"View +X variants"** or go to **Files and versions** to see all available downloads.
-
-    ## About
-
-    No changes to datasets or capabilities. Fully functional, 100% of what the original authors intended - just without the refusals.
-
-    These are meant to be the best lossless uncensored models out there.
-
-    ## Aggressive Variant
-
-    Stronger uncensoring — model is fully unlocked and won't refuse prompts. May occasionally append short disclaimers (baked into base model training, not refusals) but full content is always generated.
-
-    For a more conservative uncensor that keeps some safety guardrails, check the Balanced variant when it's available.
-
-    ## Downloads
-
-    All quants generated with importance matrix (imatrix) for optimal quality preservation on abliterated weights.
-
-    ## What are K_P quants?
-
-    ...
+    This entry installs the Q8_0 GGUF together with its F16 multimodal projector
+    for llama.cpp. The model card recommends Jinja chat templates and at least a
+    128K context for its thinking behavior. License: Apache-2.0.
   license: "apache-2.0"
   tags:
     - llm
@@ -2009,7 +2089,7 @@
   files:
     - filename: ds4flash.gguf
       uri: https://huggingface.co/unsloth/DeepSeek-V4-Flash-GGUF
-      sha256: 856c407993ccffa9ad52e23fbef8bb7b458c792a52278f4ca7931741b0c20ce2
+      sha256: ea3dc48cb9797ea1bfaa8a74d8a819756b06b16e8fbaa30728ad2cd0a643c605
 - name: "qwopus3.6-35b-a3b-coder-mtp"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -2108,6 +2188,83 @@
     - filename: llama-cpp/models/Qwen-AgentWorld-35B-A3B-GGUF/Qwen-AgentWorld-35B-A3B-UD-Q4_K_M.gguf
       sha256: e7a8eafdd8013443b6bcc4b6fb47b2d2025f772d359650b9ceb7d75971e22cad
       uri: https://huggingface.co/unsloth/Qwen-AgentWorld-35B-A3B-GGUF/resolve/main/Qwen-AgentWorld-35B-A3B-UD-Q4_K_M.gguf
+- &agents-a1-4b
+  name: "agents-a1-4b"
+  variants:
+    - model: agents-a1-4b-q8
+  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+  urls:
+    - https://huggingface.co/InternScience/Agents-A1-4B
+    - https://huggingface.co/InternScience/Agents-A1-4B-Q4_K_M-GGUF
+  description: |
+    Agents-A1-4B is InternScience's Apache-2.0 dense 4B agentic model, based on
+    Qwen3.5. It is trained for long-horizon search, engineering and scientific
+    research, instruction following, tool use, and multimodal tasks. This entry
+    uses the official Q4_K_M GGUF quantization and vision projector.
+  license: "apache-2.0"
+  tags:
+    - llm
+    - gguf
+    - vision
+    - multimodal
+    - gpu
+    - cpu
+  icon: https://huggingface.co/InternScience/Agents-A1-4B/resolve/main/figures/logo_nobg.png
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+    mmproj: llama-cpp/mmproj/Agents-A1-4B-Q4_K_M/Agents-A1-4B-mmproj.gguf
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/Agents-A1-4B-Q4_K_M/Agents-A1-4B-Q4_K_M.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/Agents-A1-4B-Q4_K_M/Agents-A1-4B-Q4_K_M.gguf
+      sha256: d93c393a9bd5139a4b5cfe24d31ef553c5a497bfb8afec178a354ecbf508f062
+      uri: huggingface://InternScience/Agents-A1-4B-Q4_K_M-GGUF/Agents-A1-4B-Q4_K_M.gguf
+    - filename: llama-cpp/mmproj/Agents-A1-4B-Q4_K_M/Agents-A1-4B-mmproj.gguf
+      sha256: 254145e7e03e9e8d3120813fac8033ffa04e411eb6d70a198833504935681084
+      uri: huggingface://InternScience/Agents-A1-4B-Q4_K_M-GGUF/Agents-A1-4B-mmproj.gguf
+- !!merge <<: *agents-a1-4b
+  name: "agents-a1-4b-q8"
+  variants: []
+  urls:
+    - https://huggingface.co/InternScience/Agents-A1-4B
+    - https://huggingface.co/InternScience/Agents-A1-4B-Q8_0-GGUF
+  description: |
+    Agents-A1-4B is InternScience's Apache-2.0 dense 4B agentic model, based on
+    Qwen3.5. It is trained for long-horizon search, engineering and scientific
+    research, instruction following, tool use, and multimodal tasks. This entry
+    uses the official Q8_0 GGUF quantization and vision projector.
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+    mmproj: llama-cpp/mmproj/Agents-A1-4B-Q8_0/Agents-A1-4B-mmproj.gguf
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/Agents-A1-4B-Q8_0/Agents-A1-4B-Q8_0.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/Agents-A1-4B-Q8_0/Agents-A1-4B-Q8_0.gguf
+      sha256: c327f66e820dae550bd230394595071c79f48c88d411b452d013ee4b5999fcea
+      uri: huggingface://InternScience/Agents-A1-4B-Q8_0-GGUF/Agents-A1-4B-Q8_0.gguf
+    - filename: llama-cpp/mmproj/Agents-A1-4B-Q8_0/Agents-A1-4B-mmproj.gguf
+      sha256: 254145e7e03e9e8d3120813fac8033ffa04e411eb6d70a198833504935681084
+      uri: huggingface://InternScience/Agents-A1-4B-Q8_0-GGUF/Agents-A1-4B-mmproj.gguf
 - name: "ornith-1.0-9b"
   variants:
     - model: ornith-1.0-9b-mtp
@@ -2631,6 +2788,83 @@
     - filename: llama-cpp/models/LFM2.5-1.2B-Instruct-GGUF/LFM2.5-1.2B-Instruct-Q4_K_M.gguf
       sha256: b1b3de114215d9507409a662a501a631095a479a419584e8a2ded6304b19b4f5
       uri: https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct-GGUF/resolve/main/LFM2.5-1.2B-Instruct-Q4_K_M.gguf
+- &lfm2-5-2-6b
+  name: "lfm2.5-2.6b"
+  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+  urls:
+    - https://huggingface.co/LiquidAI/LFM2.5-2.6B
+    - https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF
+  description: |
+    LFM2.5-2.6B is LiquidAI's compact, text-only reasoning model for on-device
+    agentic workloads. It has 2.69B parameters, a 128K-token context window,
+    multilingual support, and post-training for tool use, instruction following,
+    data extraction, RAG, and multi-step agents. This entry uses the recommended
+    Q4_K_M GGUF quantization from LiquidAI's official repository.
+  license: "other"
+  tags:
+    - llm
+    - gguf
+    - reasoning
+    - cpu
+    - gpu
+  icon: https://cdn-uploads.huggingface.co/production/uploads/61b8e2ba285851687028d395/2b08LKpev0DNEk6DlnWkY.png
+  variants:
+    - model: lfm2.5-2.6b-q8
+  overrides:
+    backend: llama-cpp
+    context_size: 131072
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+      - completion
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q4_K_M.gguf
+      repeat_penalty: 1.1
+      temperature: 0.1
+      top_k: 50
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q4_K_M.gguf
+      sha256: 79fdf00351b46cf26f020aead28d01889886be87c55fa0eb907e6f9b00bfee14
+      uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q4_K_M.gguf
+- !!merge <<: *lfm2-5-2-6b
+  name: "lfm2.5-2.6b-q8"
+  description: |
+    LFM2.5-2.6B is LiquidAI's compact, text-only reasoning model for on-device
+    agentic workloads. It has 2.69B parameters, a 128K-token context window,
+    multilingual support, and post-training for tool use, instruction following,
+    data extraction, RAG, and multi-step agents. This entry uses the higher-quality
+    Q8_0 GGUF quantization from LiquidAI's official repository.
+  variants: null
+  overrides:
+    backend: llama-cpp
+    context_size: 131072
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+      - completion
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf
+      repeat_penalty: 1.1
+      temperature: 0.1
+      top_k: 50
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/LFM2.5-2.6B-GGUF/LFM2.5-2.6B-Q8_0.gguf
+      sha256: 36587fdf27bdfc69caf2637273679a0870ec155162161bde6fd16e8c70bdb757
+      uri: https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF/resolve/main/LFM2.5-2.6B-Q8_0.gguf
 - name: "qwopus3.6-27b-coder-compat-mtp"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -2670,6 +2904,86 @@
     - filename: llama-cpp/mmproj/Qwopus3.6-27B-Coder-Compat-MTP-GGUF/mmproj-F32.gguf
       sha256: 32f7ea0600c07272547da401d460f8abbd980f3a57b69d6df87be0e2505e0b9c
       uri: https://huggingface.co/Jackrong/Qwopus3.6-27B-Coder-Compat-MTP-GGUF/resolve/main/mmproj-F32.gguf
+- &qwen3-5-9b-hauhaucs-aggressive
+  name: "qwen3.5-9b-hauhaucs-aggressive"
+  variants:
+    - model: qwen3.5-9b-hauhaucs-aggressive-q8
+  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+  urls:
+    - https://huggingface.co/Qwen/Qwen3.5-9B
+    - https://huggingface.co/HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive
+  description: |
+    Qwen3.5 9B Aggressive is HauhauCS's refusal-removed fine-tune of the
+    multimodal Qwen3.5 9B model. It retains the base model's reasoning, tool
+    use, image and video understanding, and 262K-token native context window.
+
+    This entry uses the balanced Q4_K_M GGUF quantization and includes the
+    matching BF16 multimodal projector. The Q8_0 variant offers higher fidelity.
+  license: "apache-2.0"
+  tags:
+    - llm
+    - gguf
+    - cpu
+    - gpu
+    - qwen
+    - multimodal
+    - uncensored
+  icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg
+  last_checked: "2026-08-04"
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+    mmproj: llama-cpp/mmproj/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
+      sha256: 2ca636d9e81d3d23ca9b60c234fe185d30ec082eeba69ce770fdb0c76559a4f5
+      uri: huggingface://HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
+    - filename: llama-cpp/mmproj/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+      sha256: 05f662501f8bd45607b079723a3e238a4e888fd085a10a53f4057a0e250f6934
+      uri: huggingface://HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+- !!merge <<: *qwen3-5-9b-hauhaucs-aggressive
+  name: "qwen3.5-9b-hauhaucs-aggressive-q8"
+  variants: []
+  description: |
+    Qwen3.5 9B Aggressive is HauhauCS's refusal-removed fine-tune of the
+    multimodal Qwen3.5 9B model. It retains the base model's reasoning, tool
+    use, image and video understanding, and 262K-token native context window.
+
+    This entry uses the higher-fidelity Q8_0 GGUF quantization and includes the
+    matching BF16 multimodal projector.
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+    mmproj: llama-cpp/mmproj/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf
+      sha256: 99e7f2201c0046b05d2825e4d8be6a2efad2b87b071cd55d37bdd9fbe201a58b
+      uri: huggingface://HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf
+    - filename: llama-cpp/mmproj/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+      sha256: 05f662501f8bd45607b079723a3e238a4e888fd085a10a53f4057a0e250f6934
+      uri: huggingface://HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
 # DFlash speculative-decoding pairs (upstream llama.cpp `draft-dflash`).
 # Each entry ships a full target model plus a small block-diffusion drafter
 # (z-lab DFlash, converted with upstream convert_hf_to_gguf.py, GGUF arch

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/gofrs/flock v0.13.0
 	github.com/google/go-containerregistry v0.21.6
 	github.com/google/uuid v1.6.0
-	github.com/gpustack/gguf-parser-go v0.24.0
+	github.com/gpustack/gguf-parser-go v0.25.0
 	github.com/hpcloud/tail v1.0.0
 	github.com/ipfs/go-log v1.0.5
 	github.com/jaypipes/ghw v0.24.0

go.sum

@@ -666,8 +666,8 @@ github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
-github.com/gpustack/gguf-parser-go v0.24.0 h1:tdJceXYp9e5RhE9RwVYIuUpir72Jz2D68NEtDXkKCKc=
-github.com/gpustack/gguf-parser-go v0.24.0/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
+github.com/gpustack/gguf-parser-go v0.25.0 h1:1AMBhMKtI24nTtn588Bq53FqNiOvEw1x9Nb4HbRrThs=
+github.com/gpustack/gguf-parser-go v0.25.0/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=

pkg/vram/gguf_reader.go

@@ -2,6 +2,7 @@ package vram
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	gguf "github.com/gpustack/gguf-parser-go"
@@ -10,7 +11,18 @@ import (
 
 type defaultGGUFReader struct{}
 
-func (defaultGGUFReader) ReadMetadata(ctx context.Context, uri string) (*GGUFMeta, error) {
+func (defaultGGUFReader) ReadMetadata(ctx context.Context, uri string) (meta *GGUFMeta, err error) {
+	// gguf-parser-go parses lengths supplied by the file and has historically
+	// panicked on values that cannot fit in a Go slice. Metadata can come from
+	// an untrusted remote host, and this reader is also used by a background
+	// gallery worker, where an escaped panic would terminate the whole server.
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			meta = nil
+			err = fmt.Errorf("read GGUF metadata: parser panic: %v", recovered)
+		}
+	}()
+
 	u := downloader.URI(uri)
 	urlStr := u.ResolveURL()
 
@@ -28,7 +40,10 @@ func (defaultGGUFReader) ReadMetadata(ctx context.Context, uri string) (*GGUFMet
 	if !u.LooksLikeHTTPURL() {
 		return nil, nil
 	}
-	f, err := gguf.ParseGGUFFileRemote(ctx, urlStr)
+	// The estimator only consumes architecture scalars. Tokenizer arrays can
+	// be very large and are unnecessary here, so avoid downloading or
+	// allocating them for remote files just as the local path does above.
+	f, err := gguf.ParseGGUFFileRemote(ctx, urlStr, gguf.SkipLargeMetadata())
 	if err != nil {
 		return nil, err
 	}

pkg/vram/gguf_reader_test.go

@@ -0,0 +1,115 @@
+package vram_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/LocalAI/pkg/vram"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("DefaultGGUFReader", func() {
+	It("reads architecture scalars from a valid remote GGUF", func() {
+		server := serveGGUF(validRemoteGGUF())
+
+		meta, err := vram.DefaultGGUFReader().ReadMetadata(context.Background(), server.URL+"/model.gguf")
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(meta).To(Equal(&vram.GGUFMeta{
+			BlockCount:           32,
+			EmbeddingLength:      4096,
+			HeadCount:            32,
+			HeadCountKV:          8,
+			MaximumContextLength: 8192,
+		}))
+	})
+
+	It("rejects an overflowing tokenizer array without allocating it", func() {
+		server := serveGGUF(malformedGGUFArray(math.MaxUint64))
+
+		_, err := vram.DefaultGGUFReader().ReadMetadata(context.Background(), server.URL+"/model.gguf")
+
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).NotTo(ContainSubstring("parser panic"),
+			"large tokenizer metadata should be skipped with a bounds error")
+	})
+
+	It("converts a parser panic from malformed string metadata to an error", func() {
+		server := serveGGUF(malformedGGUFString(uint64(math.MaxInt64)))
+
+		_, err := vram.DefaultGGUFReader().ReadMetadata(context.Background(), server.URL+"/model.gguf")
+
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("parser panic"))
+	})
+})
+
+func serveGGUF(payload []byte) *httptest.Server {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeContent(w, r, "model.gguf", time.Time{}, bytes.NewReader(payload))
+	}))
+	DeferCleanup(server.Close)
+	return server
+}
+
+func malformedGGUFString(length uint64) []byte {
+	payload := ggufHeader(1)
+	payload = appendGGUFString(payload, "general.name")
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeString))
+	payload = binary.LittleEndian.AppendUint64(payload, length)
+	return payload
+}
+
+func validRemoteGGUF() []byte {
+	payload := ggufHeader(6)
+	payload = appendGGUFStringValue(payload, "general.architecture", "llama")
+	payload = appendGGUFUint32(payload, "llama.block_count", 32)
+	payload = appendGGUFUint32(payload, "llama.embedding_length", 4096)
+	payload = appendGGUFUint32(payload, "llama.attention.head_count", 32)
+	payload = appendGGUFUint32(payload, "llama.attention.head_count_kv", 8)
+	payload = appendGGUFUint32(payload, "llama.context_length", 8192)
+	return payload
+}
+
+func malformedGGUFArray(itemLength uint64) []byte {
+	payload := ggufHeader(1)
+	payload = appendGGUFString(payload, "tokenizer.ggml.tokens")
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeArray))
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeString))
+	payload = binary.LittleEndian.AppendUint64(payload, 1)
+	payload = binary.LittleEndian.AppendUint64(payload, itemLength)
+	return payload
+}
+
+func ggufHeader(metadataCount uint64) []byte {
+	payload := make([]byte, 0, 128)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMagicGGUFLe))
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFVersionV3))
+	payload = binary.LittleEndian.AppendUint64(payload, 0)
+	payload = binary.LittleEndian.AppendUint64(payload, metadataCount)
+	return payload
+}
+
+func appendGGUFString(payload []byte, value string) []byte {
+	payload = binary.LittleEndian.AppendUint64(payload, uint64(len(value)))
+	return append(payload, value...)
+}
+
+func appendGGUFStringValue(payload []byte, key, value string) []byte {
+	payload = appendGGUFString(payload, key)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeString))
+	return appendGGUFString(payload, value)
+}
+
+func appendGGUFUint32(payload []byte, key string, value uint32) []byte {
+	payload = appendGGUFString(payload, key)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeUint32))
+	return binary.LittleEndian.AppendUint32(payload, value)
+}

scripts/build/backend-signing_test.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+WORKFLOW="$(dirname "$(realpath "$0")")/../../.github/workflows/backend_merge.yml"
+
+sign_commands=$(grep -Ec -- '^[[:space:]]+cosign sign([[:space:]]|$)' "$WORKFLOW" || true)
+bundle_flags=$(grep -Ec -- '^[[:space:]]+--new-bundle-format([[:space:]]|$)' "$WORKFLOW" || true)
+if [ "$sign_commands" -ne 2 ] || [ "$bundle_flags" -ne "$sign_commands" ]; then
+  echo "FAIL: every backend signing command must request the new bundle format (commands=$sign_commands flags=$bundle_flags)"
+  exit 1
+fi
+
+echo "PASS: backend signing emits Sigstore bundles for both registries"

scripts/build/llama-cpp-build-target_test.sh

@@ -29,4 +29,11 @@ assert_target arm64 "" llama-cpp-cpu-all
 assert_target amd64 sycl_f16 llama-cpp-fallback
 assert_target amd64 sycl_f32 llama-cpp-fallback
 
+# ROCm exhausts the same 6h budget through volume rather than a stall: hipcc
+# compiles ggml's HIP kernels once per AMDGPU target, eleven of them, and the
+# CPU variant matrix goes on top. 2h27m before it was added, killed at exactly
+# 6h00m on every run since.
+assert_target amd64 hipblas llama-cpp-fallback
+assert_target arm64 hipblas llama-cpp-fallback
+
 echo "PASS: llama.cpp build target preserves CPU variants where supported"

website/content/blog/what-landed-in-localai-3-10.md

@@ -0,0 +1,59 @@
+---
+title: "LocalAI 3.10: the Anthropic and Responses APIs, and one image for every GPU"
+date: 2026-01-18
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "anthropic", "open-responses", "gpu", "moonshine"]
+summary: "A /v1/messages endpoint that Claude clients can talk to unchanged, Open Responses compatibility that passes the official acceptance tests, and GPU libraries moved inside the backend containers so one image works on any hardware."
+extracss: ["blog.css"]
+---
+
+Half the tooling worth using speaks a shape of API that is not OpenAI's. You find a client you like, it talks to Anthropic, and swapping it onto a local model means either rewriting the client or gluing a translation layer in front of it. Same story with the agent frameworks that went all in on the Responses API.
+
+3.10.0 adds both surfaces natively, so the client does not have to know.
+
+## Two more front doors
+
+The Anthropic Messages API is served at `/v1/messages`, and at `/messages` for clients that do not prefix. Tool calling, streaming and non-streaming all work, so `anthropic-sdk-go`, LangChain and anything else built on that shape can be pointed at your instance without a code change.
+
+The Open Responses API is at `/v1/responses`, with `/v1/responses/:id` to fetch one and `/v1/responses/:id/cancel` to stop it. It is stateful: pass a `response_id` and the conversation resumes, set `background: true` and the agent runs asynchronously while you go and do something else, then come back for the result. Streaming covers tools, images and audio.
+
+That one passes the [official acceptance tests](https://www.openresponses.org/compliance), which was the bar I wanted to hit before shipping it.
+
+## One image for every GPU
+
+This is the change most likely to affect you even if you do not care about agents.
+
+GPU libraries (CUDA, ROCm, Vulkan) now live inside the backend containers rather than in the image you pull. There is no longer a CUDA image, a ROCm image and a CPU image to choose between. You pull the image, and acceleration works if the hardware is there! Vulkan arm64 builds are in too.
+
+It is experimental, and I want to be clear about that rather than bury it. It is a real architectural change to how every backend gets its libraries, and there will be hardware combinations we did not hit. If it does not work on yours, please file an issue, that is genuinely the most useful thing you can do for this one.
+
+## Everything else
+
+The backend gallery is system aware now, so it only lists backends your machine can actually run. No more scrolling past MLX entries on a Linux box.
+
+Tool calls stream properly, including partial arguments as `input_json_delta`, and models that emit tools as XML (`<function>...</function>`) get parsed instead of dumping the markup into the message text. Both work across llama.cpp, vLLM and diffusers.
+
+Thinking tags are extracted into a separate `reasoning` field rather than being left in the answer, in both SSE and non-SSE mode. The chat UI shows them under a Thinking tab.
+
+There is a video generation page in the web UI with LTX-2 behind it, doing text-to-video and image-to-video with the usual `fps`, `num_frames` and `guidance_scale` controls.
+
+There is request tracing now. `GET /api/traces` returns in-memory request and response logs, `/api/traces/clear` empties them. It is memory backed and drops old entries past a size cap, so it is for debugging an agent that is misbehaving right now, not for an audit trail.
+
+Two new speech backends. Moonshine is an ONNX transcription engine aimed at low-end hardware, and it is the one to reach for on a Pi or an old laptop. It is quick! Pocket-TTS does lightweight TTS with voice cloning, though the cloning path needs a HuggingFace login and a registered voice model, so it is not quite copy-paste.
+
+## Old hardware, and AMD memory
+
+Two fixes worth calling out because they were silent failures rather than errors.
+
+LocalAI was crashing on Intel CPUs without BMI2 (Sandy Bridge, Ivy Bridge), showing up as an `EOF` during model warmup rather than anything that pointed at the cause. It now falls back to `llama-cpp-fallback` on those chips.
+
+On AMD, used and total VRAM were swapped when parsing `rocm-smi` output, so a dual-Radeon box reported nonsense. `HIP_VISIBLE_DEVICES` is also handled properly now, which matters if you are pinning to the discrete GPU.
+
+## Thanks
+
+Thanks to @richiejp, @majiayu000, @nanoandrew4, @DEVMANISHOFFL, @coffeerunhobby, @rampa3, @Nold360, @jroeber and @Divyanshupandey007 for the work in this cycle.
+
+If the unified GPU backends misbehave on your setup, open an issue with what hardware you are on. And if you are wiring up the Anthropic or Responses endpoints and something does not match the spec, tell me, I would rather hear it from you than find out later.
+
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v3.10.0).

website/content/blog/what-landed-in-localai-4-0.md

@@ -0,0 +1,69 @@
+---
+title: "LocalAI 4.0: agents in the core, and a React interface"
+date: 2026-03-14
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "agents", "agenthub", "mcp", "react", "webrtc"]
+summary: "Native agent orchestration with the Agenthub, a rewritten interface with Canvas mode, MCP Apps with tool streaming, and two things removed."
+extracss: ["blog.css"]
+---
+
+Running an agent locally has meant running two things: an inference server, and a separate orchestrator that talks to it. That is a lot of moving parts for something you wanted to try on a Tuesday evening.
+
+4.0.0 puts the agent side in the core. You create agents, give them memory and skills, connect them to MCP servers, and start and stop them from the same interface you already use for models.
+
+This is a major version bump, so there are two removals near the bottom of this post. Read those before you upgrade.
+
+## Agents, and the Agenthub
+
+Agents are managed through the React interface: create one, wire up MCP servers and skills, connect it to Slack, watch what it is doing through a new Events column in the agents list.
+
+Memory has two options. Hybrid search backed by PostgreSQL if you already run one, or in-memory storage via Chromem if you do not want another service. Skills live in a central database rather than being pasted per agent.
+
+The bit I am most curious to see used is [Agenthub](https://agenthub.localai.io), a community space for sharing agent configurations. You publish one, somebody else imports it into their instance and runs it against their own models on their own hardware!
+
+## The interface is React now
+
+The web interface has been rewritten. The old one had reached the point where adding anything meant fighting it.
+
+Canvas mode is the new thing worth turning on: enable it in chat and code blocks and artifacts the model produces render in a preview pane on the right instead of scrolling past you as text. The System view splits Models and Backends into tabs. Traces render as accordions, which makes a long one readable. And if you try to install a model whose weights exceed your system RAM, you get a warning first rather than a locked-up machine.
+
+## MCP Apps
+
+Client-side MCP support is complete in this release ([#8947](https://github.com/mudler/LocalAI/pull/8947)). You pick which MCP servers to enable for a chat directly in the interface, and their tools get injected into the normal chat with streaming, so there is no separate agent mode to switch into.
+
+If you would rather not have any of it, `LOCALAI_DISABLE_MCP` turns the whole thing off.
+
+## Audio, video, and MLX across machines
+
+WebRTC is wired into the Realtime API and the Talk page ([#8790](https://github.com/mudler/LocalAI/pull/8790)), which is a real improvement for latency over what was there before.
+
+Three new audio backends: fish-speech, ace-step.cpp, and faster-qwen3-tts (CUDA only). TTS gained `sample_rate` support through post-processing, and Qwen TTS handles multiple voices.
+
+There is also an experimental MLX distributed backend for spreading a workload across Apple machines ([#8801](https://github.com/mudler/LocalAI/pull/8801)). It is early, so expect rough edges if you try it.
+
+## Infrastructure
+
+Persistent data now has its own location, separate from configuration. `LOCALAI_DATA_PATH` (or `--data-path`) points at where agents, skills, tasks, jobs and the collection database live, defaulting to `data/` under the base path. If you are mounting volumes, this is the one to look at.
+
+Shell completion scripts generate for bash, zsh and fish. There is dedicated Podman documentation now, including rootless setup.
+
+## Two things are gone
+
+The HuggingFace backend has been removed.
+
+AIO images are dropped. They existed to bundle a preset of models with the runtime, and maintaining them across every hardware variant stopped being worth what they gave people. Use the main images and install models from the gallery.
+
+## One known issue
+
+The `diffusers` backend is not in this release. It failed to build because we exhausted our CI limits, so the previous version is still what you get if you install it.
+
+This is an infrastructure problem, not a code one, and it is the kind of thing that will keep happening to us. If you know anybody at GitHub who could help us get better ARM runners, please reach out, I am not too proud to ask.
+
+## Thanks
+
+Thanks to @richiejp, @nanoandrew4, @Weathercold, @sozercan, @lukasdotcom, @loryanstrant, @bittoby and @attilagyorffy.
+
+If you build an agent worth sharing, put it on the Agenthub. The more the merrier!
+
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.0.0).

website/content/blog/what-landed-in-localai-4-1.md

@@ -0,0 +1,76 @@
+---
+title: "LocalAI 4.1: more than one box, and more than one user"
+date: 2026-04-02
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "distributed", "auth", "oidc", "quotas", "fine-tuning"]
+summary: "Distributed cluster mode that places requests by real free VRAM, OIDC with per-user API keys and quotas, and LoRA fine-tuning that exports straight to GGUF."
+extracss: ["blog.css"]
+---
+
+Two problems show up the moment LocalAI stops being a thing you run for yourself.
+
+The first is that you have more than one machine, and only one of them is doing any work. The second is that other people are using your instance, and you have no way to tell who is burning the GPU, or to stop them.
+
+4.1.0 is mostly about those two.
+
+## Running as a cluster
+
+Distributed mode lets you point several nodes at one control plane and stop thinking about which one to call.
+
+Routing orders nodes by available VRAM, so the request lands on the card with room for it. Node groups let you pin models to a subset of the cluster, which is how you keep a heavy diffusion model off the boxes doing embeddings. There is a min/max autoscaler with a reconciler managing node lifecycle, and you can drain a node for maintenance and resume it later through the API instead of pulling it out from under in-flight requests.
+
+Model transfer between nodes goes over S3 or peer to peer, so a model you have already pulled once does not have to come down from the internet again on every node!
+
+The cluster status shows up on the home page.
+
+## Users, keys and quotas
+
+LocalAI ships a multi-user platform now, which is the piece that makes it deployable for a team or a classroom rather than just for you.
+
+- User management from the React interface.
+- OIDC/OAuth against your own identity provider (Google, Keycloak, Authentik, whatever you already run).
+- Invite mode, so registration is closed unless an admin lets somebody in.
+- Per-user API keys.
+- Admin impersonation, for when somebody reports a bug you cannot reproduce.
+
+On top of that there is a quota system: set per-user limits and have them enforced, with a usage dashboard broken down per user and a predictive view of where consumption is heading.
+
+## Fine-tuning without leaving the interface
+
+Both of these are experimental. I would use them on something you can afford to throw away.
+
+Fine-tuning uses HuggingFace TRL to train LoRA adapters, exports the result to GGUF automatically, and imports it back into LocalAI so you can serve what you just trained without moving files around by hand. There is a small evals framework included to check whether the thing you trained is actually better.
+
+The quantization backend produces optimized variants of a model on the fly.
+
+## Agents from the terminal
+
+You can run an agent without the server now:
+
+```sh
+local-ai agent run <name>
+local-ai agent list
+```
+
+`run` takes an agent from the pool registry in `pool.json`, or a single-turn `--prompt` if you just want one answer. Tool calls stream in real time, and the interleaved-thinking bug that mangled output when a model reasoned mid-tool-call is fixed.
+
+## The rest of the interface work
+
+The model pipeline editor is visual, so wiring models together no longer means editing YAML. Backend logs can be scoped to a single model rather than reading the whole stream. Studio pages remember past generations, so images and audio you made last week are still there. The model and backend selectors are searchable. Error toasts link straight to the trace that produced them.
+
+## Under the hood
+
+Inference defaults are pulled from Unsloth and applied across all endpoints and gallery models, so models arrive with sane sampling parameters instead of whatever the default happened to be. `min_p` is supported. When native tool-call parsing fails, an iterative fallback parser takes over rather than returning nothing.
+
+Repeated log lines get collapsed. NVIDIA Jetson and Tegra are detected as first-class platforms. SYCL backends auto-disable `mmap`, which was crashing them on Intel GPUs. llama.cpp bundles `libdl`, `librt` and `libpthread` for portability. And the downloader rewrites HuggingFace URIs through `HF_ENDPOINT`, which is the one you need if you are behind a corporate mirror.
+
+## Thanks
+
+Thanks to @richiejp for a large chunk of this cycle, and to @tv42, @walcz-de, @majiayu000 and @ER-EPR.
+
+There is a full setup walkthrough on video if you would rather watch than read: [youtube.com/watch?v=cMVNnlqwfw4](https://www.youtube.com/watch?v=cMVNnlqwfw4).
+
+If you are setting up distributed mode or OIDC and hit a wall, reach out, I am happy to help you get it standing up.
+
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.1.0).

website/content/blog/what-landed-in-localai-4-2.md

@@ -1,21 +1,20 @@
-+++
-title = "LocalAI 4.2.0: who spoke when, and whose face is that"
-date = 2026-05-11
-description = "Speaker diarization, voice and face recognition, and an Ollama-compatible API."
-url = "/blog/localai-4-2-0/"
-+++
-
-![Diarization: segment, embed, and cluster into speaker-labelled segments](/images/diagrams/diarization-pipeline.png)
+---
+title: "LocalAI 4.2: who spoke when, and whose face is that"
+date: 2026-05-11
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "diarization", "voice-recognition", "face-recognition", "ollama", "backends"]
+summary: "A /v1/audio/diarization endpoint, voice and face recognition with liveness, a drop-in Ollama API, and eleven new backends."
+extracss: ["blog.css"]
+---
 
 You record an hour of standup, run it through Whisper, and get back one long wall of text. Every word is correct. You still have no idea who said any of them, so you end up scrubbing through the audio with the transcript open in another window, guessing at voices.
 
 4.2.0 is mostly about that class of problem. Audio and images carry more than "here are the words" or "here is a picture", and until now LocalAI had nowhere to put the rest of it.
 
-Enough chitchat, let's look at what's in it.
-
 ## Who spoke when
 
-There's a new `/v1/audio/diarization` endpoint, shaped like `/v1/audio/transcriptions` so your existing multipart code mostly carries over:
+There is a new `/v1/audio/diarization` endpoint, shaped like `/v1/audio/transcriptions` so your existing multipart code mostly carries over:
 
 ```bash
 curl http://localhost:8080/v1/audio/diarization \
@@ -37,13 +36,13 @@ curl http://localhost:8080/v1/audio/diarization \
 }
 ```
 
-Two backends serve it. [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) does pure diarization (pyannote-3.0 segmentation, a speaker-embedding extractor, then clustering) and never transcribes, so you don't pay for ASR you didn't ask for. `vibevoice-cpp` emits speaker-labelled segments as a by-product of its long-form ASR pass, so with `include_text=true` you get a transcript per segment for free! `response_format` gives you `json`, `verbose_json`, or `rttm` if you want to feed the output to `dscore`.
+Two backends serve it. [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) does pure diarization (pyannote-3.0 segmentation, a speaker-embedding extractor, then clustering) and never transcribes, so you do not pay for ASR you did not ask for. `vibevoice-cpp` emits speaker-labelled segments as a by-product of its long-form ASR pass, so with `include_text=true` you get a transcript per segment for free! `response_format` gives you `json`, `verbose_json`, or `rttm` if you want to feed the output to `dscore`.
 
 One thing to know before you build on it: `SPEAKER_00` is local to a single request. Run the same meeting twice and the numbering can come out differently, and nothing promises that `SPEAKER_00` in Monday's recording is the same human as `SPEAKER_00` in Tuesday's. If you need identity across files, pair it with `/v1/voice/embed` and keep your own embedding store. Which brings me to..
 
 ## Voices and faces
 
-`/v1/voice/*` is new: verify (are these two clips the same person?), identify (which of my enrolled speakers is this?), embed (give me the vector, I'll do the rest myself), and analyze (age, gender, emotion).
+`/v1/voice/*` is new ([#9500](https://github.com/mudler/LocalAI/pull/9500)): verify (are these two clips the same person?), identify (which of my enrolled speakers is this?), embed (give me the vector, I will do the rest myself), and analyze (age, gender, emotion).
 
 ```bash
 local-ai models install speechbrain-ecapa-tdnn
@@ -63,9 +62,9 @@ curl -sX POST http://localhost:8080/v1/voice/verify \
 
 The default threshold is around 0.25 for ECAPA-TDNN, and it moves per engine, so pass `threshold` explicitly if you swap the model out.
 
-`/v1/face/*` does the same thing for faces, plus detection and demographics, and 4.2.0 adds antispoofing. Holding a printed photo or a phone screen up to the camera is the oldest attack on face auth there is, and the liveness check rejects it.
+`/v1/face/*` does the same thing for faces ([#9480](https://github.com/mudler/LocalAI/pull/9480)), plus detection and demographics, and 4.2.0 adds antispoofing. Holding a printed photo or a phone screen up to the camera is the oldest attack on face auth there is, and the liveness check rejects it.
 
-Some honest limits. Liveness is an arms race and this is not bank-grade. The demographic heads emit confident-looking numbers for age and emotion that you should read as a rough signal and not as a fact about a person. And the default `insightface` buffalo packs are released for non-commercial research use only, so if you're shipping this in a product, pick the OpenCV Zoo entry instead. That's in the docs, but people skip docs, so it's here too.
+Some honest limits. Liveness is an arms race and this is not bank-grade. The demographic heads emit confident-looking numbers for age and emotion that you should read as a rough signal and not as a fact about a person. And the default `insightface` buffalo packs are released for non-commercial research use only, so if you are shipping this in a product, pick the OpenCV Zoo entry instead. That is in the docs, but people skip docs, so it is here too.
 
 The samples never leave your machine, which is the part I actually care about. They go from your process to the backend running next to it and nowhere else. Doing biometrics against somebody else's cloud API always felt like the worst possible trade.
 
@@ -75,21 +74,21 @@ The samples never leave your machine, which is the part I actually care about. T
 OLLAMA_HOST=http://localhost:8080 ollama run qwen3
 ```
 
-LocalAI answers the Ollama API now, so a tool that only ever learned to talk to Ollama keeps working with no code change on your side. `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags`, `/api/show`, `/api/ps` and `/api/version` all land on the engine you were already running, and your existing `/v1/*` clients are untouched.
+LocalAI answers the Ollama API now ([#9284](https://github.com/mudler/LocalAI/pull/9284)), so a tool that only ever learned to talk to Ollama keeps working with no code change on your side. `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags`, `/api/show`, `/api/ps` and `/api/version` all land on the engine you were already running, and your existing `/v1/*` clients are untouched.
 
-There's no `/api/pull` in there. Models come from the LocalAI gallery or from a URL you hand it, so `ollama run` against something you haven't installed yet won't go and fetch it for you.
+There is no `/api/pull` in there. Models come from the LocalAI gallery or from a URL you hand it, so `ollama run` against something you have not installed yet will not go and fetch it for you.
 
-## Video, and a UI repaint
+## Video, and an interface repaint
 
-`stable-diffusion.ggml` generates video now! There are gallery entries for Wan 2.1 FLF2V 14B 720P and Wan i2v 720p, including first-last-frame interpolation.
+`stable-diffusion.ggml` generates video now ([#9420](https://github.com/mudler/LocalAI/pull/9420))! There are gallery entries for Wan 2.1 FLF2V 14B 720P and Wan i2v 720p, including first-last-frame interpolation.
 
-The React UI got a long cycle of work. The chat is redesigned, the palette moved to Nord, and there's i18n across English, Italiano, Español, Deutsch and 简体中文. You can brand your instance too - name, tagline, logo, favicon - and the login page, sidebar, footer and browser tab all pick it up. Handy if you run LocalAI for a team and would rather it didn't look like somebody's side project.
+The React interface got a long cycle of work. The chat is redesigned, the palette moved to Nord, and there is i18n across English, Italiano, Español, Deutsch and 简体中文. You can brand your instance too - name, tagline, logo, favicon - and the login page, sidebar, footer and browser tab all pick it up. Handy if you run LocalAI for a team and would rather it did not look like somebody's side project.
 
 The model config editor is interactive now, with autocomplete over known fields and live validation, and it renames the file on save so you stop accumulating three copies of the same config.
 
 ## Eleven new backends
 
-sglang, ik-llama.cpp, TurboQuant, sam.cpp, Kokoros, qwen3tts.cpp, tinygrad-multimodal (experimental, don't build anything load-bearing on it yet), vibevoice.cpp, LocalVQE, insightface, and voice-rec.
+sglang, ik-llama.cpp, TurboQuant, sam.cpp, Kokoros, qwen3tts.cpp, tinygrad-multimodal (experimental, do not build anything load-bearing on it yet), vibevoice.cpp, LocalVQE, insightface, and voice-rec.
 
 vLLM reached feature parity with llama.cpp in this cycle. The full `AsyncEngineArgs` surface is exposed as a generic YAML map, and tensor-parallel distributed workers let a single model span nodes. There are CUDA 13 builds for vLLM, vLLM-omni and sglang, plus L4T arm64 for Jetson-class boards.
 
@@ -105,16 +104,14 @@ Most of the 279 pull requests here are not features. A sample of what actually w
 - faster-whisper emits word-level timestamps.
 - gfx1151 (Strix Halo / Ryzen AI MAX) works, with `AMDGPU_TARGETS` exposed as a build-arg.
 
-On the security side: an unsafe `sprintf()` came out of the C++ grpc-server, env-supplied API keys are stripped from Settings API requests before they get persisted so they can't leak back out through the config, and deleting a user on PostgreSQL cascades across everything they owned instead of leaving orphaned rows behind.
+On the security side: an unsafe `sprintf()` came out of the C++ grpc-server, env-supplied API keys are stripped from Settings API requests before they get persisted so they cannot leak back out through the config, and deleting a user on PostgreSQL cascades across everything they owned instead of leaving orphaned rows behind.
 
 Distributed mode got a hardening pass. Round-robin across replicas of the same model, "Upgrade All" scoped to the nodes that actually have the backend installed, NATS `backend.upgrade` split off from install, and correct VRAM/RAM reporting on NVIDIA unified-memory hosts.
 
-## Thanks!
+## Thanks
 
 This one had a lot of hands on it. Thanks to @richiejp for the model config editor, Kokoros and a pile of build fixes, @Anai-Guo, @russell, @leinasi2014, @keithmattix for gfx1151, @orbisai0security and @SAY-5 for the security work, @walcz-de, @thelittlefireman, @sec171, @pjbrzozowski, @mvanhorn, @arteven, @Dennisadira, @eglia, @arbrick, @neurocis and @ER-EPR.
 
-If you're wiring up diarization or the voice endpoints and get stuck, open an issue or reach out, I'm genuinely happy to help you get it working. And if LocalAI is useful to you, consider [donating](https://github.com/sponsors/mudler) or just telling somebody about it. The more the merrier!
+If you are wiring up diarization or the voice endpoints and get stuck, open an issue or reach out, I am genuinely happy to help you get it working.
 
-[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.2.0). See [Speaker diarization]({{% relref "features/audio-diarization" %}}), [Voice recognition]({{% relref "features/voice-recognition" %}}) and [Face recognition]({{% relref "features/face-recognition" %}}).
-
-Cheers!
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.2.0).

website/content/blog/what-landed-in-localai-4-3.md

@@ -0,0 +1,105 @@
+---
+title: "LocalAI 4.3: signed backends, and the prompt cache that was off"
+date: 2026-05-24
+author: "Ettore Di Giacinto"
+category: "Release"
+tags: ["release", "security", "cosign", "prompt-cache", "distributed", "usage"]
+summary: "Keyless cosign verification for backend OCI images, the llama.cpp prompt cache enabled by default, per-API-key usage attribution, and the replica-pinning bug that kept a second node idle."
+extracss: ["blog.css"]
+---
+
+Here is a gap that had been sitting in LocalAI for a while. The gallery YAML tells LocalAI which OCI image to pull for a backend, and then LocalAI pulls it. Nothing checked that the bytes coming back were the bytes we built. A compromised registry, or somebody in the middle, and you would never know.
+
+4.3.0 closes that, and fixes a default that had been quietly costing everybody a lot of prefill time.
+
+## Signed backends
+
+Every backend image merged by CI is now signed with [sigstore](https://www.sigstore.dev/)/cosign, keyless via Fulcio and Rekor, including each per-arch entry under the manifest list ([#9823](https://github.com/mudler/LocalAI/pull/9823)). It uses OCI 1.1 referrers rather than the legacy `:tag.sig` convention.
+
+On your side, verification runs against a policy that the gallery declares:
+
+```yaml
+verification:
+  issuer_regex: "^https://token\\.actions\\.githubusercontent\\.com$"
+  identity_regex: "^https://github\\.com/mudler/LocalAI/\\.github/workflows/backend_merge\\.yml@.*$"
+  not_before: "2026-05-22T00:00:00Z"
+```
+
+A few details that took some thinking.
+
+`not_before` is the revocation lever. Keyless Fulcio certificates are ephemeral, so there is nothing to revoke on the signing side. Revocation has to be policy side: move the date forward in the gallery YAML and every signature older than it stops validating.
+
+The TUF trusted root is cached process-wide, so installing ten backends from one gallery does one fetch instead of ten.
+
+Digest pinning closes the window between verifying and pulling, which is otherwise a TOCTOU you could drive a truck through.
+
+Strict mode is `--require-backend-integrity`, or `LOCALAI_REQUIRE_BACKEND_INTEGRITY=true`. It turns a missing policy or an empty SHA256 from a warning into a hard failure.
+
+Now the honest part: strict mode is opt-in and off by default, and until a gallery ships a `verification:` block, installs go through with a warning. The default `backend/index.yaml` does not have the blocks populated yet, that is the next step. So today this is machinery that works and is not yet enforcing much. Turn on strict mode in production once your gallery is populated, not before, or you will just break your own installs.
+
+## The prompt cache was off
+
+`llama-cpp` has a server-side prompt cache. LocalAI was not enabling it. So every agent turn, every coding-assistant call, every OpenAI-compatible CLI with a long system prompt, re-prefilled that whole prompt from scratch.
+
+On the reported workload, a repeated system prompt took 5 to 8 minutes per call before this change and seconds after it. Your numbers will depend on how long your prompt is and what hardware you are on.
+
+Two defaults flipped ([#9925](https://github.com/mudler/LocalAI/pull/9925), [#9951](https://github.com/mudler/LocalAI/pull/9951)):
+
+1. `kv_unified` is now `true` in `grpc-server.cpp`. The old `false` was silently force-disabling `cache_idle_slots` at server init, so the host prompt cache got allocated and then never written across requests. That is the one that actually explains the behaviour.
+2. `prompt_cache_all` defaults to `true` at the YAML layer, matching upstream llama.cpp's own default in `common.h`. The per-request `cache_prompt` knob is on out of the box.
+
+You can opt out with `options: ["kv_unified:false"]` or `prompt_cache_all: false`, and there are new keys (`cache_idle_slots`, `checkpoint_every_nt`) if you want to tune it. The model configuration docs got a worked example for the repeated-system-prompt case and an explanation of how `kv_unified`, `cache_ram` and `cache_idle_slots` interact, because they interact in ways that are not obvious.
+
+## Who is burning the GPU
+
+The usage page could tell you how many tokens were spent. It could not tell you who spent them ([#9920](https://github.com/mudler/LocalAI/pull/9920)).
+
+`usage_records` gained a `Source` column (`apikey`, `web`, `legacy`) plus the API key id and name, with an idempotent backfill of older rows on `InitDB`. The auth middleware passes the resolved key and the request source through, and usage middleware snapshots the key id and name at write time, so a key you revoke later still reads correctly in history (it renders as `(revoked)` rather than vanishing).
+
+Two new endpoints:
+
+```
+GET /api/auth/usage/sources        # your own
+GET /api/auth/admin/usage/sources  # everyone, with user_id / api_key_id filters
+```
+
+The admin view truncates at 200 keys. The React usage page gained a Sources tab with a source-mix ribbon, a top-7-plus-Other time chart, and a sortable table. Web interface session traffic is split per user instead of being lumped into one global row.
+
+## Distributed v3, and one good bug
+
+This one is worth writing down because the symptom and the cause were far apart.
+
+An operator reported this:
+
+```
+dgx-spark1     loaded   in_flight=6
+nvidia-thor1   loaded   in_flight=0
+```
+
+Two replicas of the same model, one taking everything, one idle forever. The round-robin was there and looked correct.
+
+The cause: `ModelLoader.Load` cached a `*Model` whose embedded `InFlightTrackingClient` was bound to a single `(nodeID, replicaIndex)`. The first request picked a node and got wrapped. Every request after that reused the wrapper, so it kept going to whichever node won the first pick, even after the reconciler scaled the model out. The routing code was fine. It just was not being consulted again!
+
+`SmartRouter.Route` now runs per request ([#9968](https://github.com/mudler/LocalAI/pull/9968)), the `in_flight ASC, last_used ASC, available_vram DESC` ordering actually fires, and replica selection lives in one place (`PickBestReplica`) with a spec asserting the SQL `ORDER BY` and the Go picker agree on a seeded dataset. `probeHealth` is memoized per `(nodeID, addr)` with a 30 second TTL and `singleflight` coalescing, because llama.cpp serializes `HealthCheck` against in-flight `Predict` and a burst of new requests would otherwise stall on it.
+
+Two other distributed changes.
+
+`POST /api/nodes/:id/backends/install` used to block for up to 3 minutes while the worker pulled the image, which froze the Backends picker in the interface. It returns HTTP 202 and a `jobID` immediately now ([#9928](https://github.com/mudler/LocalAI/pull/9928)). Install and upgrade timeouts are configurable via `LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT` and `LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT`, defaulting to 15 minutes instead of the hardcoded 3. A NATS round-trip timeout while the worker is still pulling reports as `running_on_worker` rather than a hard failure.
+
+Workers also publish debounced install progress (~250ms) that the master forwards into the operations status ([#9958](https://github.com/mudler/LocalAI/pull/9958)), so distributed installs show per-byte progress the same way local ones do. Old workers stay silent and new masters tolerate the silence, so mixed-version clusters keep working.
+
+## Smaller things
+
+`LOCALAI_TRACING_MAX_BODY_BYTES` caps trace payload size, which stops the admin Traces page from trying to render a 40 MB embedding response.
+
+There is a `flake.nix` with a dev shell for NixOS users who do not want to go through Docker.
+
+The `vllm`, `sglang` and `vllm-omni` L4T13 backends are back for Jetson and DGX boxes, switched to PyPI aarch64+cu130 wheels to fix the torch 2.10 ABI mismatch.
+
+A distributed test harness landed in `tests/distributed/`, aimed at catching the class of regression the replica-pinning bug belonged to.
+
+## Thanks
+
+If you run LocalAI in production, the two things to look at here are strict mode (once your gallery has a `verification:` block) and whether the prompt cache change speeds up your workload. I would like to hear numbers from real setups, mine are one data point.
+
+[Full release notes](https://github.com/mudler/LocalAI/releases/tag/v4.3.0).

website/content/blog/what-landed-in-localai-4-8.md

@@ -1,14 +1,14 @@
 ---
 title: "What landed in LocalAI 4.8"
-date: 2026-08-01
+date: 2026-08-04
 author: "Ettore Di Giacinto"
 category: "Release"
-tags: ["release", "vllm.cpp", "audio.cpp", "3d", "gallery", "distributed", "performance"]
-summary: "A new inference engine, 3D generation, one backend that serves six audio endpoints, and a web interface 3.48x lighter. 321 pull requests in eighteen days."
+tags: ["release", "vllm.cpp", "audio.cpp", "3d", "agent", "gallery", "distributed", "performance"]
+summary: "A new inference engine, a terminal agent in the CLI, 3D generation, and a web interface 3.48x lighter. 386 pull requests in twenty-two days."
 extracss: ["blog.css"]
 ---
 
-LocalAI 4.8.0 is out. It took eighteen days and 321 merged pull requests, and it pulls in two directions at once: three new things LocalAI can do that it could not do before, and a long list of places where it now does the old things without lying to you.
+LocalAI 4.8.0 is out, after twenty-two days and 386 merged pull requests. There are four new things LocalAI can do, and a lot of repair work on things it already did.
 
 The full notes list everything. This post covers the parts that change what you do day to day, with the pull request numbers so you can read the diffs.
 
@@ -36,6 +36,11 @@ The third one was `/api/traces` returning a 21 MB unpaginated blob that the UI p
 
 ## One gallery entry, several builds
 
+<figure>
+<img src="/media/v4-8-0-ui-model-variants.png" alt="The model detail pane listing every variant">
+<figcaption>One entry, four builds. LocalAI picks the largest that fits and marks it auto-selected.</figcaption>
+</figure>
+
 Installing a model no longer means reading a list of quantizations and guessing which one your card will hold. A gallery entry can now declare `variants:`, a list of references to other entries that are alternative builds of the same weights:
 
 ```yaml
@@ -55,12 +60,43 @@ Every surface can override the choice: `variant` on `POST /models/apply`, `local
 
 One gap worth knowing about: in distributed mode `InstallModel` resolves against the frontend rather than the worker that will serve the model, so a cluster with a small frontend and large workers selects conservatively. PRs [#10943](https://github.com/mudler/LocalAI/pull/10943), [#10983](https://github.com/mudler/LocalAI/pull/10983), [#10992](https://github.com/mudler/LocalAI/pull/10992), [#11027](https://github.com/mudler/LocalAI/pull/11027) and [#11139](https://github.com/mudler/LocalAI/pull/11139).
 
-## A new engine: vllm.cpp
+## A new engine: vllm.cpp (alpha)
 
-[vllm.cpp](https://github.com/mudler/vllm.cpp) is a from-scratch C++20 port of vLLM, written and maintained by the LocalAI team under Apache-2.0, and it ships here as the `vllm-cpp` backend ([#11100](https://github.com/mudler/LocalAI/pull/11100)). It mirrors vLLM's V1 architecture, so paged KV cache, continuous batching, prefix caching, scheduler and sampler, on a portable tensor runtime with no Python, no PyTorch and no ggml at inference. It loads Hugging Face safetensors and GGUF, enforces structured output inside the engine (JSON schema, regex, choice, GBNF), and builds for CPU amd64 and arm64, CUDA 12 and 13 including Blackwell, L4T for GB10, Vulkan and Darwin Metal.
+[vllm.cpp](https://github.com/mudler/vllm.cpp) is Apache-2.0 and maintained by the LocalAI team. We want it community-first rather than a LocalAI-only engine, so it lives in its own repository with its own docs, benchmark record and issue tracker, and it runs without LocalAI anywhere in the picture. It began as a C++20 port of vLLM. It ships here as the `vllm-cpp` backend ([#11100](https://github.com/mudler/LocalAI/pull/11100)). It implements vLLM's V1 architecture, so paged KV cache, continuous batching, prefix caching, scheduler and sampler, on a portable tensor runtime with no Python, no PyTorch and no ggml at inference. vLLM stays its reference implementation: correctness is checked by comparing output against it, and the benchmark scoreboard is kept against it.
+
+It has grown features vLLM does not have, which is most of the reason the port exists. It loads GGUF as well as safetensors, runs on CPU, Apple Metal and Vulkan alongside CUDA 12 and 13 and L4T for GB10, and ships speculative decoding and KV offload. Its benchmark page now measures against llama.cpp, MLX-LM and DwarfStar as well as vLLM, because on that hardware those are the engines it competes with. The project is expected to be renamed, with the new name still to be decided; it is drifting far enough that vllm.cpp will eventually mislead.
 
 Tool calling is at llama.cpp parity by construction, because chat deliberately reuses the same autoparser path: full minja chat templates, `tool_choice: auto` lowered to a lazy structural-tag decode constraint, 30 tool dialects, 7 reasoning parsers, and streamed `ChatDelta` and `ToolCallDelta`.
 
+<figure>
+<img src="/media/v4-8-0-vllm-cpp-scoreboard.png" alt="Throughput of vllm.cpp relative to each reference engine, drawn as deviation from parity">
+<figcaption>llama.cpp is left out because its 1.18x is a prefill ratio, and putting that on the same axis as throughput would compare two different measurements.</figcaption>
+</figure>
+
+Numbers from the project's own [scoreboard](https://github.com/mudler/vllm.cpp/blob/master/docs/BENCHMARKS.md), which calls ties ties and losses losses. Above 1.0 means vllm.cpp is ahead:
+
+<div class="tw">
+<table>
+<thead><tr><th>Reference</th><th>Workload</th><th>Result</th></tr></thead>
+<tbody>
+<tr><td>vLLM</td><td>Qwen3.6-27B NVFP4, GB10</td><td>1.045x at concurrency 1, 1.007x to 1.017x from c2 to c32, output token-for-token identical</td></tr>
+<tr><td>vLLM</td><td>Qwen3.6-35B-A3B NVFP4, GB10</td><td>1.010x at c16 and 1.013x at c32, behind from c1 to c8 (0.817x at c1)</td></tr>
+<tr><td>llama.cpp</td><td>Qwen3.5-2B GGUF, CPU aarch64</td><td>prefill 1.18x, decode a tie, memory parity</td></tr>
+<tr><td>MLX-LM</td><td>Qwen3-0.6B, Apple M4</td><td>97.6% of warm total, prefill ahead</td></tr>
+<tr><td>DwarfStar (ds4)</td><td>DeepSeek-V4-Flash IQ2_XXS, one DGX Spark</td><td>18.69 vs 16.33 tok/s decode, <b>1.144x</b>, same output</td></tr>
+<tr><td>vLLM</td><td>Laguna-XS-2.1 NVFP4, GB10</td><td>44.46 vs 43.10 tok/s, <b>1.03x</b>, same output</td></tr>
+</tbody>
+</table>
+</div>
+
+The upstream page is careful about its own noise: on the 27B grid the run-to-run spread is 0.5% and c2 through c32 land between 0.7% and 1.7%, so it calls those five ties rather than wins. The concurrency-1 result is the one it stands behind.
+
+The DeepSeek-V4-Flash row is the one that shows how far this has moved from being a vLLM port. It runs DeepSeek-V4-Flash at roughly 2-bit (IQ2_XXS mixed, about 80 GB) on a single DGX Spark, decoding at 18.69 tok/s against DwarfStar's 16.33. At 300B+ total parameters even a 4-bit checkpoint is 156 GB or more, so a 2-bit GGUF is what fits inside the Spark's 119 GiB unified pool, and reading GGUF is what makes that possible.
+
+That number moved twice in a week, and the second move came from one lever. The dense Q8_0 projection tower was being read from the GGUF mmap over unified memory, which the GB10 reads about 20% slower per-GEMV than device memory. Staging that 6 GiB tower device-resident once at load, same bytes and same kernels, took decode from 16.23 to 18.69, generating the same tokens and using no more peak memory. The same change took Laguna-XS-2.1 from 87% of vLLM to 1.03x ahead of it.
+
+Speculative decoding is in similar shape: MTP on Qwen3.6-27B NVFP4 generates the same tokens as vLLM's MTP and runs about 4% faster at concurrency 1.
+
 Configuration is a normal backend install:
 
 ```yaml
@@ -73,9 +109,24 @@ options:
 - max_num_seqs:16      # also: block_size:<n>, num_blocks:<n>
 ```
 
-The CPU path is verified end to end against `Qwen3.5-2B-UD-Q8_K_XL.gguf` with the full Ginkgo suite, covering blocking and streaming byte-parity, greedy determinism, stop words, GBNF-constrained generation, concurrent streams, reasoning split and both `required` and `auto` tool calls. The maturity statement from the release notes is worth repeating in full:
+**Treat these as alpha development builds, not a released backend.** vllm.cpp is early, and shipping it in 4.8 is about getting it in front of people who want to try it, not about recommending it for anything you care about. `llama-cpp` stays the default for real use.
 
-> The GPU images build and ship, but their runtime behavior has not been through the same e2e gate yet. This is a first release of a young engine: no throughput comparison against upstream vLLM is claimed here, and `llama-cpp` remains the default recommendation for general use. Try it, and please report what breaks.
+The CPU path is verified end to end against `Qwen3.5-2B-UD-Q8_K_XL.gguf` with the full Ginkgo suite, covering blocking and streaming byte-parity, greedy determinism, stop words, GBNF-constrained generation, concurrent streams, reasoning split and both `required` and `auto` tool calls. The GPU images build and ship, but their runtime behavior has not been through that gate. No throughput comparison against upstream vLLM is claimed. Expect rough edges, and please report what breaks.
+
+On Apple Silicon the image now ships vllm.cpp's MLX GEMM provider ([#11137](https://github.com/mudler/LocalAI/pull/11137)). Upstream keeps it off by default because it adds about 124 MB, so we measured before turning it on. Qwen3-1.7B-bf16 on an M4, p=512 g=128, both arms toggled on one binary so a build difference cannot explain the gap:
+
+<div class="tw">
+<table>
+<thead><tr><th>Batch</th><th>MLX tok/s</th><th>native tok/s</th><th>speedup</th><th>MLX TTFT</th><th>native TTFT</th></tr></thead>
+<tbody>
+<tr><td>1</td><td>5.79</td><td>3.08</td><td><b>1.88x</b></td><td>3.32 s</td><td>7.68 s</td></tr>
+<tr><td>4</td><td>15.75</td><td>10.24</td><td><b>1.54x</b></td><td>9.63 s</td><td>18.77 s</td></tr>
+<tr><td>16</td><td>38.65</td><td>17.69</td><td><b>2.19x</b></td><td>18.33 s</td><td>54.48 s</td></tr>
+</tbody>
+</table>
+</div>
+
+Two reps, with rep spread reaching 9.4%, so treat the multipliers as +/-10%. Time to first token roughly halves across the range.
 
 <figure>
 <video src="/media/vllm-race.mp4" muted loop playsinline preload="none" data-lazy aria-label="vllm.cpp generating tokens"></video>
@@ -84,7 +135,7 @@ The CPU path is verified end to end against `Qwen3.5-2B-UD-Q8_K_XL.gguf` with th
 
 ## LocalAI generates 3D models now
 
-This is a new modality rather than a new backend under an existing one, so it goes through the whole stack: a `Generate3D` RPC in `backend.proto`, a `FLAG_3D` capability so the loader knows which backends can serve it, and `POST /v1/3d/generations`.
+3D generation is a new modality, so it had to be wired through the whole stack: a `Generate3D` RPC in `backend.proto`, a `FLAG_3D` capability so the loader knows which backends can serve it, and `POST /v1/3d/generations`.
 
 The first engine behind it is `trellis2cpp`, an image-to-3D backend over TRELLIS.2. You give it an image, you get a GLB back. The web UI has a page for it with a native GLB viewer, so you can turn the result around in the browser instead of downloading it to find out whether it worked, history kept in IndexedDB so a reload does not lose your generations, and previewable print remeshing for output you actually intend to send to a printer ([#10979](https://github.com/mudler/LocalAI/pull/10979)).
 
@@ -93,9 +144,23 @@ The first engine behind it is `trellis2cpp`, an image-to-3D backend over TRELLIS
 <figcaption>trellis2-4b, 2,502,928 vertices and 5,012,118 triangles, turning in the browser. The remesh slider below it is the print path.</figcaption>
 </figure>
 
+## `local-ai chat` stopped being a REPL
+
+`local-ai chat` used to be a chat prompt in a terminal. It is now an agent, and it is the [nib](https://github.com/mudler/nib) harness compiled straight into the binary: tool use behind an approval gate, sub-agents, MCP servers, plugins and skills, auto-configured against your own instance. Nothing extra to install.
+
+```bash
+local-ai chat                      # the agent, pointed at your models
+echo "what is 2+2" | local-ai chat --cli
+local-ai chat --init zsh           # Ctrl+Space from any shell prompt
+```
+
+That last one prints a shell integration script (zsh, bash or fish), so you can pull the agent up from wherever you already are instead of opening something else.
+
+It runs shell commands now, so every tool call goes through an approval prompt you control, and read-only ones like `ls` and `cat` run without asking. If you had habits around the old REPL, a few things moved: `/clear` is gone and `/compact` is the closest thing, `/models` and `/model <name>` mean what they always meant, and switching model keeps the conversation instead of starting over ([#11291](https://github.com/mudler/LocalAI/pull/11291)).
+
 ## One backend, six audio endpoints
 
-The usual shape for audio is one backend per model family, which means a process per capability and a config file for each. `audio-cpp` wraps [audio.cpp](https://github.com/0xShug0/audio.cpp), a multi-family ggml audio engine, and inverts that: one backend process serves several unrelated families through a single runtime vocabulary, and works out which family a checkpoint belongs to from the GGUF's own `audiocpp.model_spec.family` metadata key. There is nothing backend-specific to write in the model config.
+The usual shape for audio is one backend per model family, which means a process per capability and a config file for each. `audio-cpp` wraps [audio.cpp](https://github.com/0xShug0/audio.cpp), a multi-family ggml audio engine. One backend process serves several unrelated families through a single runtime vocabulary, and works out which family a checkpoint belongs to from the GGUF's own `audiocpp.model_spec.family` metadata key. There is nothing backend-specific to write in the model config.
 
 <div class="tw">
 <table>
@@ -130,7 +195,12 @@ The `bonsai` backend serves the 1-bit (Q1_0) and ternary (Q2_0) Bonsai quantizat
 
 ## The operations bar became a page
 
-The old operations bar rendered one row per in-flight operation above every page. Queue four model installs and a backend and it took most of the viewport, on every route, until the last one finished. Two things were conflated there: a global "something is happening" signal, which needs one line, and the detail of what is happening, which needs somewhere to put it.
+<figure>
+<img src="/media/v4-8-0-ui-activity.png" alt="The Activity page with four installs running">
+<figcaption>Four backend installs in flight, and the record of what already finished.</figcaption>
+</figure>
+
+The old operations bar rendered one row per in-flight operation above every page. Queue four model installs and a backend and it took most of the viewport, on every route, until the last one finished. It was doing two jobs at once. A global "something is happening" signal only needs one line, and the detail of what is happening needs a page of its own.
 
 The strip is now one line, permanently, showing a failure first and otherwise the least-advanced running operation, with a `+N more` pill. Its `✕` hides the strip and no longer cancels anything. That is a deliberate behavior change worth knowing about before you click it out of habit: the same glyph used to cancel a 17 GB download in one row and dismiss a message in the next. Cancelling moved to the new page, behind a button that says so.
 
@@ -179,6 +249,6 @@ Valkey Search joins the vector store options as the `valkey-store` backend ([#11
 
 This is also the release where localai.io split in two: the project site at the root, and the documentation under `/docs/`. Every URL that was published before still resolves, through 214 generated redirect stubs, because GitHub Pages has no server-side rewrites to do it properly ([#11243](https://github.com/mudler/LocalAI/pull/11243)).
 
-Twenty-four people contributed to this release, eleven of them for the first time. The gallery went from 1,221 entries to 1,505.
+Twenty-five people contributed to this release, eleven of them for the first time. The gallery went from 1,221 entries to 1,515.
 
 To upgrade, pull `localai/localai:latest` or re-run the install script. The [full changelog](https://github.com/mudler/LocalAI/compare/v4.7.1...v4.8.0) has everything this post left out.

website/layouts/index.html

@@ -19,7 +19,7 @@
           <div><b class="tnum" data-count="{{ .Site.Data.stats.stars }}">0</b><span>GitHub stars</span></div>
           <div><b class="tnum" data-count="73">0</b><span>Backends</span></div>
           <div><b class="tnum" data-count="{{ len .Site.Data.engines.engines }}">0</b><span>Engines we wrote</span></div>
-          <div><b class="tnum" data-count="1585">0</b><span>Models, one click</span></div>
+          <div><b class="tnum" data-count="1255">0</b><span>Models, one click</span></div>
         </div>
       </div>
       <div class="fd">
@@ -39,7 +39,8 @@
     <p class="kicker rv">The runtime</p>
     <h2 class="rv mt1" style="max-width:21ch">Everything else plugs into LocalAI.</h2>
     <p class="lede rv mt2">One binary with an OpenAI-compatible API in front of it. Point an existing client at it and the calls keep working, except now the model is on your machine. It also speaks the Anthropic, Ollama and ElevenLabs APIs, so most tools need a URL change and nothing else.</p>
-    <p class="lede rv mt2">Underneath, a small core pulls each engine in as a separate backend, only when a model asks for it. That is why one install covers this much ground without becoming a 9 GB download.</p>
+    <p class="lede rv mt2">The engine behind that API is swappable. One model can run on llama.cpp while the next loads on vLLM, SGLang or MLX, and the client never notices: same endpoint, same request, different engine underneath. Switching is one line in the model's config.</p>
+    <p class="lede rv mt2">A small core pulls each engine in as a separate backend, only when a model asks for it. That is why one install covers this much ground without becoming a 9 GB download.</p>
     <div class="apis rv">
       <span>OpenAI API</span><span>Anthropic API</span><span>Ollama API</span><span>ElevenLabs API</span><span>Realtime over WebRTC</span>
     </div>
@@ -57,7 +58,7 @@
       </div>
       <div class="duo__m rv">
         <figure class="screen" style="margin:0">
-          <figcaption class="screen__bar"><i></i> localai · model gallery <b>1,585 models</b></figcaption>
+          <figcaption class="screen__bar"><i></i> localai · model gallery <b>1,255 models</b></figcaption>
           <video src="/media/gallery.mp4" muted loop playsinline preload="none" data-lazy aria-label="Installing a model from the LocalAI gallery"></video>
         </figure>
       </div>
@@ -327,7 +328,7 @@
   <div class="shell">
     <div class="bars rv" aria-hidden="true"><i></i><i></i><i></i><i></i></div>
     <p class="kicker rv">The gallery</p>
-    <h2 class="rv mt1" style="max-width:20ch">1,585 models. No notebook, no conversion script.</h2>
+    <h2 class="rv mt1" style="max-width:20ch">1,255 models. No notebook, no conversion script.</h2>
     <div class="cards">
       <a class="cd rv" href="/docs/getting-started/models/"><p class="cd__k">Quantizations</p><h3>201 APEX builds</h3>
         <p>Every tier of every model we quantize, ranked against the hardware you actually have and installed with one click.</p><span class="cd__go">Browse the gallery →</span></a>

website/static/media/v4-8-0-vllm-cpp-scoreboard.html

@@ -0,0 +1,100 @@
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<style>
+  /* palette lifted from the two logos:
+     LocalAI  #0E2632 navy, #385360 slate, #469AAF teal, #90A8AE haze
+     vllm.cpp #3AB4CA teal, #95C4D1 light                              */
+  :root{
+    --bg:#0b1c25; --ink:#e8f1f4; --dim:#90a8ae; --faint:#5d757f;
+    --teal:#3ab4ca; --teal-hi:#7fd4e2; --amber:#e0a944; --rule:#1d3440;
+  }
+  *{margin:0;padding:0;box-sizing:border-box}
+  html,body{width:1600px;height:900px}
+  body{
+    background:radial-gradient(1250px 720px at 80% -12%, #143140 0%, var(--bg) 62%);
+    color:var(--ink);
+    font-family:-apple-system,"SF Pro Display","Segoe UI",Helvetica,Arial,sans-serif;
+    -webkit-font-smoothing:antialiased; padding:58px 84px; position:relative;
+  }
+  .eyebrow{display:flex;align-items:center;gap:14px;color:var(--teal);
+    font-weight:600;font-size:23px;letter-spacing:.14em;text-transform:uppercase}
+  .eyebrow .dot{width:11px;height:11px;border-radius:50%;background:var(--teal);
+    box-shadow:0 0 16px 2px var(--teal)}
+  h1{font-size:56px;line-height:1.06;font-weight:760;margin:16px 0 6px;letter-spacing:-.02em}
+  h1 .grad{background:linear-gradient(92deg,var(--teal),var(--teal-hi));
+    -webkit-background-clip:text;background-clip:text;color:transparent}
+  .sub{color:var(--dim);font-size:23px;margin-bottom:14px}
+  svg{width:100%;height:auto;display:block}
+  .foot{position:absolute;left:84px;right:84px;bottom:40px;display:flex;
+    justify-content:space-between;align-items:center;color:var(--faint);
+    font-size:21px;border-top:1px solid var(--rule);padding-top:16px}
+  .foot .link{color:var(--ink);font-weight:600}
+</style>
+</head>
+<body>
+  <div class="eyebrow"><span class="dot"></span>vllm.cpp &middot; throughput vs the reference engine</div>
+  <h1>Measured against <span class="grad">what each workload actually runs on</span></h1>
+  <div class="sub">Throughput relative to the reference. 1.00 is parity, bars run from it. Higher is faster.</div>
+
+  <svg id="c" viewBox="0 0 1432 585"></svg>
+
+  <div class="foot">
+    <span class="link">github.com/mudler/vllm.cpp</span>
+    <span>GB10 unless noted &middot; greedy, reference in its own production config &middot; docs/BENCHMARKS.md</span>
+  </div>
+
+<script>
+const rows = [
+  {ref:'DwarfStar (ds4)', work:'DeepSeek-V4-Flash IQ2_XXS', v:1.144, note:'18.69 vs 16.33 tok/s'},
+  {ref:'vLLM',            work:'Qwen3.6-27B NVFP4, c1',     v:1.045, note:'86.05 vs 82.32 tok/s'},
+  {ref:'vLLM',            work:'Laguna-XS-2.1 NVFP4',       v:1.030, note:'44.46 vs 43.10 tok/s'},
+  {ref:'vLLM',            work:'Qwen3.6-35B-A3B, c32',      v:1.013, note:'3030.5 vs 2993.0 tok/s'},
+  {ref:'MLX-LM',          work:'Qwen3-0.6B, Apple M4',      v:0.976, note:'97.6% of warm total'},
+];
+
+const W=1432, H=585;
+const AX=64;                   // axis strip reserved at the bottom
+const LBL=470;                 // left label gutter
+const R=150;                   // right gutter for the value
+const lo=-0.055, hi=0.165;     // deviation domain around parity
+const pw=W-LBL-R;
+const x = d => LBL + pw*((d-lo)/(hi-lo));
+const zero = x(0);
+const rowH = (H-AX)/rows.length;
+const barH = 46;
+
+let g='';
+// faint engineering grid at 2% steps
+for(let d=-0.04; d<=0.16001; d+=0.02){
+  const gx=x(d), on0=Math.abs(d)<1e-9;
+  g+=`<line x1="${gx}" y1="4" x2="${gx}" y2="${H-AX+10}" stroke="${on0?'#4a6b78':'#16303c'}" stroke-width="${on0?2:1}"/>`;
+  g+=`<text x="${gx}" y="${H-22}" fill="${on0?'#90a8ae':'#4d6570'}" font-size="17" text-anchor="middle"
+       font-weight="${on0?'700':'400'}">${(1+d).toFixed(2)}</text>`;
+}
+
+rows.forEach((r,i)=>{
+  const cy = i*rowH + rowH/2;
+  const d  = r.v-1;
+  const ahead = d>=0;
+  const col = ahead ? '#3ab4ca' : '#e0a944';
+  const x0 = ahead ? zero : x(d);
+  const w  = Math.abs(x(d)-zero);
+
+  // reference + workload, two weights on one line
+  g+=`<text x="${LBL-26}" y="${cy-4}" fill="#e8f1f4" font-size="25" font-weight="670" text-anchor="end">${r.ref}</text>`;
+  g+=`<text x="${LBL-26}" y="${cy+22}" fill="#5d757f" font-size="19" text-anchor="end">${r.work}</text>`;
+
+  g+=`<rect x="${x0}" y="${cy-barH/2}" width="${Math.max(w,2)}" height="${barH}" rx="4" fill="${col}" opacity="0.92"/>`;
+
+  // value, then the raw measurement under it
+  const vx = ahead ? x(d)+18 : zero+18;
+  g+=`<text x="${vx}" y="${cy+1}" fill="${col}" font-size="27" font-weight="700"
+       font-variant-numeric="tabular-nums">${r.v.toFixed(3)}&times;</text>`;
+  g+=`<text x="${vx}" y="${cy+23}" fill="#5d757f" font-size="17">${r.note}</text>`;
+});
+document.getElementById('c').innerHTML=g;
+</script>
+</body>
+</html>