fix(mlx): strip file:// LocalPrefix before loading filesystem-imported models

MLX backends passed request.Model verbatim to mlx_lm/mlx_vlm load(). For a
model imported from the filesystem, LocalAI hands the backend a file:// URI
(its LocalPrefix), which load() rejects: the scheme is neither a valid HF
repo id nor an existing path (Path(model).exists() fails on the scheme),
producing "Repo id must be in the form 'repo_name' or 'namespace/repo_name'
... Use repo_type argument if needed".

Add a pure, unit-testable resolve_model_path(model, model_file) helper in the
shared python_utils: it prefers the resolved ModelFile, strips a file://
scheme and percent-decodes the path, and leaves plain repo ids and local
paths untouched. Wire it into the mlx, mlx-vlm and mlx-distributed backends
(load, model_key, and the distributed broadcast all use the normalized path).

Fixes #7461.

Assisted-by: claude:claude-opus-4-8 [Claude Code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.github/backend-matrix.yml

@@ -703,6 +703,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-locate-anything-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1543,6 +1556,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-locate-anything-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1569,6 +1595,19 @@ include:
     backend: "rfdetr-cpp"
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    skip-drivers: 'false'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-locate-anything-cpp'
+    base-image: "ubuntu:24.04"
+    ubuntu-version: '2404'
+    runs-on: 'ubuntu-24.04-arm'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1766,20 +1805,6 @@ include:
     dockerfile: "./backend/Dockerfile.llama-cpp"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: 'hipblas'
-    cuda-major-version: ""
-    cuda-minor-version: ""
-    platforms: 'linux/amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-gpu-rocm-hipblas-turboquant'
-    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64'
-    runs-on: 'ubuntu-latest'
-    base-image: "rocm/dev-ubuntu-24.04:7.2.1"
-    skip-drivers: 'false'
-    backend: "turboquant"
-    dockerfile: "./backend/Dockerfile.turboquant"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: 'hipblas'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -2820,6 +2845,74 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  # locate-anything-cpp
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-locate-anything-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'sycl_f32'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-intel-sycl-f32-locate-anything-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+    skip-drivers: 'false'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'sycl_f16'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-intel-sycl-f16-locate-anything-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+    skip-drivers: 'false'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-locate-anything-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-locate-anything-cpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'sycl_f32'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -2913,6 +3006,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2204'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    skip-drivers: 'false'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-arm64-locate-anything-cpp'
+    base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+    runs-on: 'ubuntu-24.04-arm'
+    backend: "locate-anything-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2204'
   # whisper
   - build-type: ''
     cuda-major-version: ""
@@ -4355,6 +4461,10 @@ includeDarwin:
     tag-suffix: "-metal-darwin-arm64-silero-vad"
     build-type: "metal"
     lang: "go"
+  - backend: "sherpa-onnx"
+    tag-suffix: "-metal-darwin-arm64-sherpa-onnx"
+    build-type: "metal"
+    lang: "go"
   - backend: "local-store"
     tag-suffix: "-metal-darwin-arm64-local-store"
     build-type: "metal"
@@ -4362,3 +4472,6 @@ includeDarwin:
   - backend: "llama-cpp-quantization"
     tag-suffix: "-metal-darwin-arm64-llama-cpp-quantization"
     build-type: "mps"
+  - backend: "speaker-recognition"
+    tag-suffix: "-metal-darwin-arm64-speaker-recognition"
+    build-type: "mps"

.github/workflows/bump_deps.yaml

@@ -62,6 +62,10 @@ jobs:
             variable: "RFDETR_VERSION"
             branch: "main"
             file: "backend/go/rfdetr-cpp/Makefile"
+          - repository: "mudler/locate-anything.cpp"
+            variable: "LOCATEANYTHING_VERSION"
+            branch: "master"
+            file: "backend/go/locate-anything-cpp/Makefile"
           - repository: "predict-woo/qwen3-tts.cpp"
             variable: "QWEN3TTS_CPP_VERSION"
             branch: "main"

.github/workflows/test-extra.yml

@@ -38,6 +38,7 @@ jobs:
       acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
       qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }}
       rfdetr-cpp: ${{ steps.detect.outputs.rfdetr-cpp }}
+      locate-anything-cpp: ${{ steps.detect.outputs.locate-anything-cpp }}
       vibevoice-cpp: ${{ steps.detect.outputs.vibevoice-cpp }}
       localvqe: ${{ steps.detect.outputs.localvqe }}
       voxtral: ${{ steps.detect.outputs.voxtral }}
@@ -563,7 +564,7 @@ jobs:
       - name: Run e2e-backends smoke
         env:
           BACKEND_IMAGE: quay.io/go-skynet/local-ai-backends:master-cpu-llama-cpp
-          BACKEND_TEST_CAPS: health,load,predict,stream,logprobs,logit_bias
+          BACKEND_TEST_CAPS: health,load,predict,stream,logprobs,logit_bias,tokenize
         run: |
           make test-extra-backend
   # Realtime e2e with sherpa-onnx driving VAD + STT + TTS against a mocked LLM.
@@ -901,6 +902,45 @@ jobs:
       - name: Test rfdetr-cpp
         run: |
           make --jobs=5 --output-sync=target -C backend/go/rfdetr-cpp test
+  # Per-backend e2e for locate-anything-cpp: builds the .so + Go binary and
+  # runs `make -C backend/go/locate-anything-cpp test`. test.sh fetches the
+  # locate-anything-q8_0 GGUF (~6.3 GB, NVIDIA LocateAnything-3B) from the
+  # published mudler/locate-anything.cpp-gguf HF repo + a COCO image, then the
+  # Go wire test loads the model and runs an open-vocabulary Detect, asserting
+  # at least one labeled box. Heavier than the other Go backends (it is a 3B),
+  # so it is gated to changes under backend/go/locate-anything-cpp/.
+  tests-locate-anything-cpp:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.locate-anything-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake curl libopenblas-dev
+      - name: Setup Go
+        uses: actions/setup-go@v5
+      - name: Display Go version
+        run: go version
+      - name: Proto Dependencies
+        run: |
+          # Install protoc
+          curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \
+          unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
+          rm protoc.zip
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
+          PATH="$PATH:$HOME/go/bin" make protogen-go
+      - name: Build locate-anything-cpp
+        run: |
+          make --jobs=5 --output-sync=target -C backend/go/locate-anything-cpp
+      - name: Test locate-anything-cpp
+        run: |
+          make --jobs=5 --output-sync=target -C backend/go/locate-anything-cpp test
   # Per-backend smoke for vibevoice-cpp: builds the .so + Go binary and
   # runs `make -C backend/go/vibevoice-cpp test`. test.sh auto-downloads
   # the published mudler/vibevoice.cpp-models bundle (TTS Q8_0 + ASR Q4_K

Dockerfile

@@ -108,6 +108,7 @@ RUN <<EOT bash
         apt-get update && \
         apt-get install -y --no-install-recommends \
             cuda-nvcc-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
+            cuda-nvrtc-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcufft-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcurand-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcublas-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \

Makefile

@@ -180,7 +180,7 @@ osx-signed: build
 
 ## Run
 run: ## run local-ai
-	CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) run ./
+	CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) run ./cmd/local-ai
 
 prepare-test: protogen-go build-mock-backend
 
@@ -566,6 +566,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/speaker-recognition
 	$(MAKE) -C backend/rust/kokoros kokoros-grpc
 	$(MAKE) -C backend/go/rfdetr-cpp
+	$(MAKE) -C backend/go/locate-anything-cpp
 
 test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/transformers test
@@ -593,6 +594,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/speaker-recognition test
 	$(MAKE) -C backend/rust/kokoros test
 	$(MAKE) -C backend/go/rfdetr-cpp test
+	$(MAKE) -C backend/go/locate-anything-cpp test
 
 ##
 ## End-to-end gRPC tests that exercise a built backend container image.

README.md

@@ -149,6 +149,16 @@ local-ai run https://gist.githubusercontent.com/.../phi-2.yaml
 local-ai run oci://localai/phi-2:latest
 ```
 
+To test a running LocalAI server from the terminal, open an interactive chat session from another shell. Inside the prompt, `/models` lists installed models and `/model <name>` switches between them.
+
+```bash
+# Terminal 1
+local-ai run llama-3.2-1b-instruct:q4_k_m
+
+# Terminal 2
+local-ai chat --model llama-3.2-1b-instruct:q4_k_m
+```
+
 > **Automatic Backend Detection**: LocalAI automatically detects your GPU capabilities and downloads the appropriate backend. For advanced options, see [GPU Acceleration](https://localai.io/features/gpu-acceleration/).
 
 For more details, see the [Getting Started guide](https://localai.io/basics/getting_started/).

backend/Dockerfile.golang

@@ -206,6 +206,16 @@ RUN if [ "${BACKEND}" = "opus" ]; then \
     apt-get clean && rm -rf /var/lib/apt/lists/*; \
 fi
 
+# CrispASR's piper TTS backend dlopens libespeak-ng at runtime to phonemize
+# non-English text (the MIT-clean path; English uses a built-in G2P). Install
+# the espeak-ng runtime + its libpcaudio/libsonic deps + voice data so
+# package.sh can bundle them into the FROM scratch image.
+RUN if [ "${BACKEND}" = "crispasr" ]; then \
+    apt-get update && apt-get install -y --no-install-recommends \
+        espeak-ng-data libespeak-ng1 libpcaudio0 libsonic0 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*; \
+fi
+
 COPY . /LocalAI
 
 RUN git config --global --add safe.directory /LocalAI

backend/Dockerfile.python

@@ -126,6 +126,7 @@ RUN <<EOT bash
         apt-get update && \
         apt-get install -y --no-install-recommends \
             cuda-nvcc-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
+            cuda-nvrtc-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcufft-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcurand-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
             libcublas-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \

backend/cpp/ds4/CMakeLists.txt

@@ -60,10 +60,12 @@ elseif(DS4_GPU STREQUAL "cpu")
     set(DS4_OBJS "${DS4_DIR}/ds4_cpu.o")
 endif()
 
-# ds4.c now references ds4_distributed.c (distributed inference was split into
-# its own translation unit upstream). It is a single GPU-agnostic object shared
-# by every GPU mode, so link it in regardless of DS4_GPU.
+# ds4.c now references ds4_distributed.c (distributed inference) and ds4_ssd.c
+# (SSD expert-cache), each split into its own translation unit upstream. Both
+# are GPU-agnostic objects shared by every GPU mode, so link them in regardless
+# of DS4_GPU.
 list(APPEND DS4_OBJS "${DS4_DIR}/ds4_distributed.o")
+list(APPEND DS4_OBJS "${DS4_DIR}/ds4_ssd.o")
 
 add_executable(${TARGET}
     grpc-server.cpp

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=477c0e82e2699b35a65fd0a1ed6fe66b41087dfe
+# Upstream pin lives below as DS4_VERSION?=d881f2a05e8ff6bec001315a36b794b4aa310173
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=477c0e82e2699b35a65fd0a1ed6fe66b41087dfe
+DS4_VERSION?=d881f2a05e8ff6bec001315a36b794b4aa310173
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -18,19 +18,20 @@ UNAME_S := $(shell uname -s)
 
 CMAKE_ARGS ?= -DCMAKE_BUILD_TYPE=Release
 
-# ds4_distributed.o is a GPU-agnostic translation unit that ds4.c/ds4_cpu.o now
-# reference (upstream split distributed inference into its own .c). The same
-# object is shared by every GPU mode, so it is appended unconditionally below.
+# ds4_distributed.o and ds4_ssd.o are GPU-agnostic translation units that
+# ds4.c/ds4_cpu.o now reference (upstream split distributed inference and the
+# SSD expert-cache into their own .c files). Both objects are shared by every
+# GPU mode, so they are appended unconditionally below.
 ifeq ($(BUILD_TYPE),cublas)
     CMAKE_ARGS += -DDS4_GPU=cuda
-    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o
+    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_ssd.o
 else ifeq ($(UNAME_S),Darwin)
     CMAKE_ARGS += -DDS4_GPU=metal
-    DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o
+    DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o ds4_ssd.o
 else
     # CPU reference path (Linux only - macOS CPU path is broken by VM bug per ds4 README).
     CMAKE_ARGS += -DDS4_GPU=cpu
-    DS4_OBJ_TARGET := ds4_cpu.o ds4_distributed.o
+    DS4_OBJ_TARGET := ds4_cpu.o ds4_distributed.o ds4_ssd.o
 endif
 
 ifneq ($(NATIVE),true)
@@ -55,11 +56,11 @@ ds4:
 # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA).
 ds4/ds4.o: ds4
 ifeq ($(BUILD_TYPE),cublas)
-	+$(MAKE) -C ds4 ds4.o ds4_cuda.o ds4_distributed.o
+	+$(MAKE) -C ds4 ds4.o ds4_cuda.o ds4_distributed.o ds4_ssd.o
 else ifeq ($(UNAME_S),Darwin)
-	+$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o
+	+$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_ssd.o
 else
-	+$(MAKE) -C ds4 ds4_cpu.o ds4_distributed.o
+	+$(MAKE) -C ds4 ds4_cpu.o ds4_distributed.o ds4_ssd.o
 endif
 
 grpc-server: ds4/ds4.o

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=1520eda980564241434b791ce2bbbd128c4be9ea
+IK_LLAMA_VERSION?=e6f8112f3ba126eed3ff5b30cdd08085414a7516
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=7c158fbb4aec1bdc9c81d6ca0e785139f4826fae
+LLAMA_VERSION?=4c6595503fe45d5a39f88d194e270f64c7424677
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -381,6 +381,15 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
             });
     }
 
+    // for each video in the request, add the video data
+    for (int i = 0; i < predict->videos_size(); i++) {
+        data["video_data"].push_back(json
+            {
+                {"id", i},
+                {"data",    predict->videos(i)},
+            });
+    }
+
     data["stop"] = predict->stopprompts();
     // data["n_probs"] = predict->nprobs();
     //TODO: images,
@@ -482,23 +491,13 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
     if (!request->draftmodel().empty()) {
         params.speculative.draft.mparams.path = request->draftmodel();
         // Default to draft type if a draft model is set but no explicit type.
-        // Upstream (post ggml-org/llama.cpp#22838) made the speculative type a
-        // vector; the turboquant fork still uses the legacy scalar. The
-        // LOCALAI_LEGACY_LLAMA_CPP_SPEC macro is injected by
-        // backend/cpp/turboquant/patch-grpc-server.sh for fork builds only.
-        // Upstream renamed COMMON_SPECULATIVE_TYPE_DRAFT -> ..._DRAFT_SIMPLE
-        // in ggml-org/llama.cpp#22964; the fork still uses the old name.
-#ifdef LOCALAI_LEGACY_LLAMA_CPP_SPEC
-        if (params.speculative.type == COMMON_SPECULATIVE_TYPE_NONE) {
-            params.speculative.type = COMMON_SPECULATIVE_TYPE_DRAFT;
-        }
-#else
+        // Upstream made the speculative type a vector (ggml-org/llama.cpp#22838)
+        // and renamed COMMON_SPECULATIVE_TYPE_DRAFT -> ..._DRAFT_SIMPLE (#22964).
         const bool no_spec_type = params.speculative.types.empty() ||
             (params.speculative.types.size() == 1 && params.speculative.types[0] == COMMON_SPECULATIVE_TYPE_NONE);
         if (no_spec_type) {
             params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE };
         }
-#endif
     }
 
     //  params.model_alias ??
@@ -574,9 +573,10 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
     // tokens (0 disables the minimum). Match upstream's default (256). This
     // field was renamed from `checkpoint_every_nt` in llama.cpp; the semantics
     // also shifted from a fixed cadence to a minimum spacing. The turboquant
-    // fork branched before the field existed, so skip it on the legacy path
-    // (LOCALAI_LEGACY_LLAMA_CPP_SPEC is injected by patch-grpc-server.sh).
-#ifndef LOCALAI_LEGACY_LLAMA_CPP_SPEC
+    // fork still lacks common_params::checkpoint_min_step, so skip it there
+    // (LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP is injected by
+    // backend/cpp/turboquant/patch-grpc-server.sh).
+#ifndef LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP
     params.checkpoint_min_step = 256;
 #endif
 
@@ -752,7 +752,7 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
                 params.cache_idle_slots = false;
             }
 
-#ifndef LOCALAI_LEGACY_LLAMA_CPP_SPEC
+#ifndef LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP
         // --- minimum context-checkpoint spacing (upstream -cms / --checkpoint-min-step) ---
         // 0 disables the minimum-spacing gate. Old option names (`checkpoint_every_nt`,
         // `checkpoint_every_n_tokens`) are kept as aliases for backward compatibility
@@ -906,17 +906,6 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
 
         // Speculative decoding options
         } else if (!strcmp(optname, "spec_type") || !strcmp(optname, "speculative_type")) {
-#ifdef LOCALAI_LEGACY_LLAMA_CPP_SPEC
-            // Fork only knows a single scalar `type`. Take the first comma-
-            // separated value and assign it via the singular helper.
-            std::string first = optval_str;
-            const auto comma = first.find(',');
-            if (comma != std::string::npos) first = first.substr(0, comma);
-            auto type = common_speculative_type_from_name(first);
-            if (type != COMMON_SPECULATIVE_TYPE_COUNT) {
-                params.speculative.type = type;
-            }
-#else
             // Upstream switched to a vector of types (comma-separated for multi-type
             // chaining via common_speculative_types_from_names). We keep accepting a
             // single value here, but also tolerate comma-separated lists.
@@ -945,7 +934,6 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
             if (!parsed.empty()) {
                 params.speculative.types = parsed;
             }
-#endif
         } else if (!strcmp(optname, "spec_n_max") || !strcmp(optname, "draft_max")) {
             if (optval != NULL) {
                 try { params.speculative.draft.n_max = std::stoi(optval_str); } catch (...) {}
@@ -983,21 +971,6 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
             // shares the target context size. Accept the option for backward
             // compatibility but silently ignore it.
 
-// Everything below relies on struct shape introduced in ggml-org/llama.cpp#22838
-// (parallel drafting): `ngram_mod`, `ngram_map_k`, `ngram_map_k4v`,
-// `ngram_cache`, and the `draft.{cache_type_*, cpuparams*, tensor_buft_overrides}`
-// fields. The turboquant fork branched before that, so its build defines
-// LOCALAI_LEGACY_LLAMA_CPP_SPEC via patch-grpc-server.sh and these option
-// keys become unrecognized (silently dropped, like any unknown opt) for it.
-//
-// The `#ifdef LOCALAI_LEGACY_LLAMA_CPP_SPEC` / `#else` split below sits at the
-// closing-brace position of the `draft_ctx_size` branch on purpose: in the
-// legacy build the chain ends here (the brace closes draft_ctx_size), and in
-// the modern build the chain continues with `} else if (...)` instead, so the
-// brace count stays balanced under both branches of the preprocessor.
-#ifdef LOCALAI_LEGACY_LLAMA_CPP_SPEC
-        }
-#else
         // --- ngram_mod family (upstream --spec-ngram-mod-*) ---
         } else if (!strcmp(optname, "spec_ngram_mod_n_min")) {
             if (optval != NULL) {
@@ -1127,7 +1100,6 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
             }
             if (!cur.empty()) flush(cur);
         }
-#endif // LOCALAI_LEGACY_LLAMA_CPP_SPEC — closes the `else`/`#ifdef` opened at draft_ctx_size
     }
 
     // Set params.n_parallel from environment variable if not set via options (fallback)
@@ -1177,15 +1149,11 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
             params.tensor_buft_overrides.push_back({nullptr, nullptr});
         }
     }
-    // The draft tensor_buft_overrides are only populated under the modern
-    // (post-#22838) layout, whose population code is itself gated by
-    // LOCALAI_LEGACY_LLAMA_CPP_SPEC above. The turboquant fork lacks
-    // common_params_speculative::draft entirely, so skip the sentinel there too.
-#ifndef LOCALAI_LEGACY_LLAMA_CPP_SPEC
+    // Terminate the draft tensor_buft_overrides list with a sentinel, mirroring
+    // the main-model handling above.
     if (!params.speculative.draft.tensor_buft_overrides.empty()) {
         params.speculative.draft.tensor_buft_overrides.push_back({nullptr, nullptr});
     }
-#endif
 
     // TODO: Add yarn
 
@@ -1544,7 +1512,7 @@ public:
                     msg_json["role"] = msg.role();
 
                     bool is_last_user_msg = (i == last_user_msg_idx);
-                    bool has_images_or_audio = (request->images_size() > 0 || request->audios_size() > 0);
+                    bool has_images_or_audio = (request->images_size() > 0 || request->audios_size() > 0 || request->videos_size() > 0);
 
                     // Handle content - can be string, null, or array
                     // For multimodal content, we'll embed images/audio from separate fields
@@ -1595,6 +1563,16 @@ public:
                                     content_array.push_back(audio_chunk);
                                 }
                             }
+                            if (request->videos_size() > 0) {
+                                for (int j = 0; j < request->videos_size(); j++) {
+                                    json video_chunk;
+                                    video_chunk["type"] = "input_video";
+                                    json input_video;
+                                    input_video["data"] = request->videos(j);
+                                    video_chunk["input_video"] = input_video;
+                                    content_array.push_back(video_chunk);
+                                }
+                            }
                             msg_json["content"] = content_array;
                         } else {
                             // Use content as-is (already array or not last user message)
@@ -1629,6 +1607,16 @@ public:
                                 content_array.push_back(audio_chunk);
                             }
                         }
+                        if (request->videos_size() > 0) {
+                            for (int j = 0; j < request->videos_size(); j++) {
+                                json video_chunk;
+                                video_chunk["type"] = "input_video";
+                                json input_video;
+                                input_video["data"] = request->videos(j);
+                                video_chunk["input_video"] = input_video;
+                                content_array.push_back(video_chunk);
+                            }
+                        }
                         msg_json["content"] = content_array;
                     } else if (msg.role() == "tool") {
                         // Tool role messages must have content field set, even if empty
@@ -2080,6 +2068,16 @@ public:
                         files.push_back(decoded_data);
                     }
                 }
+
+                const auto &video_data = data.find("video_data");
+                if (video_data != data.end() && video_data->is_array())
+                {
+                    for (const auto &video : *video_data)
+                    {
+                        auto decoded_data = base64_decode(video["data"].get<std::string>());
+                        files.push_back(decoded_data);
+                    }
+                }
             }
 
             const bool has_mtmd = ctx_server.impl->mctx != nullptr;
@@ -2332,7 +2330,7 @@ public:
                     }
 
                     bool is_last_user_msg = (i == last_user_msg_idx);
-                    bool has_images_or_audio = (request->images_size() > 0 || request->audios_size() > 0);
+                    bool has_images_or_audio = (request->images_size() > 0 || request->audios_size() > 0 || request->videos_size() > 0);
 
                     // Handle content - can be string, null, or array
                     // For multimodal content, we'll embed images/audio from separate fields
@@ -2385,6 +2383,16 @@ public:
                                     content_array.push_back(audio_chunk);
                                 }
                             }
+                            if (request->videos_size() > 0) {
+                                for (int j = 0; j < request->videos_size(); j++) {
+                                    json video_chunk;
+                                    video_chunk["type"] = "input_video";
+                                    json input_video;
+                                    input_video["data"] = request->videos(j);
+                                    video_chunk["input_video"] = input_video;
+                                    content_array.push_back(video_chunk);
+                                }
+                            }
                             msg_json["content"] = content_array;
                         } else {
                             // Use content as-is (already array or not last user message)
@@ -2424,6 +2432,16 @@ public:
                                 content_array.push_back(audio_chunk);
                             }
                         }
+                        if (request->videos_size() > 0) {
+                            for (int j = 0; j < request->videos_size(); j++) {
+                                json video_chunk;
+                                video_chunk["type"] = "input_video";
+                                json input_video;
+                                input_video["data"] = request->videos(j);
+                                video_chunk["input_video"] = input_video;
+                                content_array.push_back(video_chunk);
+                            }
+                        }
                         msg_json["content"] = content_array;
                         SRV_INF("[CONTENT DEBUG] Predict: Message %d created content array with media\n", i);
                     } else if (!msg.tool_calls().empty()) {
@@ -2886,6 +2904,16 @@ public:
                         files.push_back(decoded_data);
                     }
                 }
+
+                const auto &video_data = data.find("video_data");
+                if (video_data != data.end() && video_data->is_array())
+                {
+                    for (const auto &video : *video_data)
+                    {
+                        auto decoded_data = base64_decode(video["data"].get<std::string>());
+                        files.push_back(decoded_data);
+                    }
+                }
             }
 
             // process files
@@ -3458,7 +3486,7 @@ public:
         if (body.count("prompt") != 0) {
             const bool add_special = json_value(body, "add_special", false);
 
-            llama_tokens tokens = tokenize_mixed(ctx_server.impl->vocab, body.at("content"), add_special, true);
+            llama_tokens tokens = tokenize_mixed(ctx_server.impl->vocab, body.at("prompt"), add_special, true);
 
 
             for (const auto& token : tokens) {

backend/cpp/turboquant/Makefile

@@ -1,7 +1,7 @@
 
 # Pinned to the HEAD of feature/turboquant-kv-cache on https://github.com/TheTom/llama-cpp-turboquant.
 # Auto-bumped nightly by .github/workflows/bump_deps.yaml.
-TURBOQUANT_VERSION?=5aeb2fdbe26cd4c534c6fa15de73cb5749bd0403
+TURBOQUANT_VERSION?=7d9715f1f071fa07c7b2ad3dbfd320b314139e65
 LLAMA_REPO?=https://github.com/TheTom/llama-cpp-turboquant
 
 CMAKE_ARGS?=

backend/cpp/turboquant/patch-grpc-server.sh

@@ -4,21 +4,19 @@
 #
 #   1. Augment the kv_cache_types[] allow-list so `LoadModel` accepts the
 #      fork-specific `turbo2` / `turbo3` / `turbo4` cache types.
-#   2. Replace `get_media_marker()` (added upstream in ggml-org/llama.cpp#21962,
-#      server-side random per-instance marker) with the legacy "<__media__>"
-#      literal. The fork branched before that PR, so server-common.cpp has no
-#      get_media_marker symbol. The fork's mtmd_default_marker() still returns
-#      "<__media__>", and Go-side tooling falls back to that sentinel when the
-#      backend does not expose media_marker, so substituting the literal keeps
-#      behavior identical on the turboquant path.
-#   3. Revert the `common_params_speculative` field references to the
-#      pre-refactor flat layout. Upstream ggml-org/llama.cpp#22397 split the
-#      struct into nested `draft` / `ngram_simple` / `ngram_mod` / etc. members;
-#      the turboquant fork branched before that PR and still exposes the flat
-#      `n_max`, `mparams_dft`, `ngram_size_n`, ... fields. The substitutions
-#      below map the new nested paths back to the legacy flat names so the
-#      shared grpc-server.cpp keeps compiling against the fork's common.h.
-#      Drop this block once the fork rebases past #22397.
+#   2. Define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP at the top of the file
+#      so the grpc-server option parser skips the two references to
+#      common_params::checkpoint_min_step (the default and the option handler).
+#      That field does not exist in the fork yet; drop this once it does.
+#
+# The fork used to lag upstream on the whole common_params_speculative refactor
+# (ggml-org/llama.cpp#22397/#22838/#22964), the model_tgt rename (#22838) and
+# get_media_marker (#21962), which required a much larger compat shim here
+# (flat-field sed renames + a coarse LOCALAI_LEGACY_LLAMA_CPP_SPEC define). The
+# fork has since rebased past all of those, so the only remaining gap is
+# checkpoint_min_step. If a future bump reintroduces a divergence, add a narrow
+# guard in grpc-server.cpp keyed on a fork-specific macro and inject it here
+# rather than resurrecting the coarse one.
 #
 # We patch the *copy* sitting in turboquant-<flavor>-build/, never the original
 # under backend/cpp/llama-cpp/, so the stock llama-cpp build keeps compiling
@@ -72,72 +70,20 @@ else
     echo "==> KV allow-list patch OK"
 fi
 
-if grep -q 'get_media_marker()' "$SRC"; then
-    echo "==> patching $SRC to replace get_media_marker() with legacy \"<__media__>\" literal"
-    # Only one call site today (ModelMetadata), but replace all occurrences to
-    # stay robust if upstream adds more. Use a temp file to avoid relying on
-    # sed -i portability (the builder image uses GNU sed, but keeping this
-    # consistent with the awk block above).
-    sed 's/get_media_marker()/"<__media__>"/g' "$SRC" > "$SRC.tmp"
-    mv "$SRC.tmp" "$SRC"
-    echo "==> get_media_marker() substitution OK"
+# 2. Define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP at the top of the file so
+#    the grpc-server option parser skips the two references to
+#    common_params::checkpoint_min_step (the default assignment and the option
+#    handler). That field does not exist in the fork yet. Drop this block once
+#    the fork rebases past the bump that added checkpoint_min_step.
+if grep -q '^#define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP' "$SRC"; then
+    echo "==> $SRC already defines LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP, skipping"
 else
-    echo "==> $SRC has no get_media_marker() call, skipping media-marker patch"
-fi
-
-if grep -q 'params\.speculative\.draft\.\|params\.speculative\.ngram_simple\.' "$SRC"; then
-    echo "==> patching $SRC to revert common_params_speculative refs to pre-#22397 flat layout"
-    # Each substitution is the exact post-refactor path → legacy flat field.
-    # Order doesn't matter because the source paths are disjoint, but we keep
-    # the most-specific (mparams.path) first for readability.
-    sed -E \
-        -e 's/params\.speculative\.draft\.mparams\.path/params.speculative.mparams_dft.path/g' \
-        -e 's/params\.speculative\.draft\.n_max/params.speculative.n_max/g' \
-        -e 's/params\.speculative\.draft\.n_min/params.speculative.n_min/g' \
-        -e 's/params\.speculative\.draft\.p_min/params.speculative.p_min/g' \
-        -e 's/params\.speculative\.draft\.p_split/params.speculative.p_split/g' \
-        -e 's/params\.speculative\.draft\.n_gpu_layers/params.speculative.n_gpu_layers/g' \
-        -e 's/params\.speculative\.draft\.n_ctx/params.speculative.n_ctx/g' \
-        -e 's/params\.speculative\.ngram_simple\.size_n/params.speculative.ngram_size_n/g' \
-        -e 's/params\.speculative\.ngram_simple\.size_m/params.speculative.ngram_size_m/g' \
-        -e 's/params\.speculative\.ngram_simple\.min_hits/params.speculative.ngram_min_hits/g' \
-        "$SRC" > "$SRC.tmp"
-    mv "$SRC.tmp" "$SRC"
-    echo "==> speculative field rename OK"
-else
-    echo "==> $SRC has no post-#22397 speculative field refs, skipping spec rename patch"
-fi
-
-# 4. Revert the `ctx_server.impl->model_tgt` rename introduced by upstream
-#    ggml-org/llama.cpp#22838 (parallel drafting). The turboquant fork still
-#    exposes the field as `model` on `server_context_impl`. The two call sites
-#    are in the Rerank and ModelMetadata RPC handlers.
-if grep -q 'ctx_server\.impl->model_tgt' "$SRC"; then
-    echo "==> patching $SRC to revert ctx_server.impl->model_tgt -> ctx_server.impl->model"
-    sed -E 's/ctx_server\.impl->model_tgt/ctx_server.impl->model/g' "$SRC" > "$SRC.tmp"
-    mv "$SRC.tmp" "$SRC"
-    echo "==> model_tgt rename OK"
-else
-    echo "==> $SRC has no ctx_server.impl->model_tgt refs, skipping model_tgt rename patch"
-fi
-
-# 5. Define LOCALAI_LEGACY_LLAMA_CPP_SPEC at the top of the file so the
-#    grpc-server option parser skips the new option-handler blocks (ngram_mod,
-#    ngram_map_k, ngram_map_k4v, ngram_cache, draft.cache_type_*, draft.cpuparams*,
-#    draft.tensor_buft_overrides) introduced for the post-#22838 layout, the
-#    draft.tensor_buft_overrides sentinel termination, and the
-#    common_params::checkpoint_min_step default/option (added with the
-#    35c9b1f3 bump). Those blocks reference struct fields that simply do not
-#    exist in the fork.
-if grep -q '^#define LOCALAI_LEGACY_LLAMA_CPP_SPEC' "$SRC"; then
-    echo "==> $SRC already defines LOCALAI_LEGACY_LLAMA_CPP_SPEC, skipping"
-else
-    echo "==> patching $SRC to define LOCALAI_LEGACY_LLAMA_CPP_SPEC at the top"
-    # Insert the define before the very first `#include` so it precedes all the
-    # speculative-decoding code paths.
+    echo "==> patching $SRC to define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP at the top"
+    # Insert the define before the very first `#include` so it precedes the
+    # checkpoint_min_step references.
     awk '
         !done && /^#include/ {
-            print "#define LOCALAI_LEGACY_LLAMA_CPP_SPEC 1"
+            print "#define LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP 1"
             print "// ^ injected by backend/cpp/turboquant/patch-grpc-server.sh"
             print ""
             done = 1
@@ -145,13 +91,13 @@ else
         { print }
         END {
             if (!done) {
-                print "patch-grpc-server.sh: no #include anchor found to insert LOCALAI_LEGACY_LLAMA_CPP_SPEC" > "/dev/stderr"
+                print "patch-grpc-server.sh: no #include anchor found to insert LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP" > "/dev/stderr"
                 exit 1
             }
         }
     ' "$SRC" > "$SRC.tmp"
     mv "$SRC.tmp" "$SRC"
-    echo "==> LOCALAI_LEGACY_LLAMA_CPP_SPEC define OK"
+    echo "==> LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK"
 fi
 
 echo "==> all patches applied"

backend/cpp/turboquant/patches/0001-hip-guard-copy2d-peer-fastpath.patch

@@ -0,0 +1,55 @@
+hip: port the turboquant CUDA additions that ggml's HIP shim doesn't cover
+
+The turboquant fork adds/modifies a few ggml-cuda.cu spots with CUDA APIs
+that ggml's HIP (and MUSA) compatibility layer does not provide, breaking
+the -gpu-rocm-hipblas-turboquant build:
+
+  1. ggml_cuda_copy2d_across_devices() (host-staged cross-device copy for
+     split mul_mat output) uses the CUDA 3D-peer copy APIs
+     cudaMemcpy3DPeerParms / make_cudaPitchedPtr / make_cudaExtent /
+     cudaMemcpy3DPeerAsync. HIP genuinely does not support these (see the
+     fork's own comment "HIP does not support cudaMemcpy3DPeerAsync"), so
+     guard the peer fast path with #if !defined(GGML_USE_HIP) &&
+     !defined(GGML_USE_MUSA) -- matching how the fork already guards the
+     same API for the sibling 2D copy -- and fall through to the existing
+     cudaMemcpyAsync staging fallback below (functionally identical,
+     slightly slower on multi-GPU ROCm).
+
+  2. ggml_backend_cuda_device_event_new() creates its event with plain
+     cudaEventCreate, which ggml's HIP shim does not alias (it only aliases
+     cudaEventCreateWithFlags). Use cudaEventCreateWithFlags(..., 
+     cudaEventDisableTiming) -- exactly what the rest of this file already
+     does (cf. lines ~1034, ~3461) and HIP-safe.
+
+CUDA builds are unaffected. Drop the relevant hunk once the fork HIP-ports
+these; apply-patches.sh fails fast if an anchor goes stale.
+
+diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
+index 0427e6b..6352e6a 100644
+--- a/ggml/src/ggml-cuda/ggml-cuda.cu
++++ b/ggml/src/ggml-cuda/ggml-cuda.cu
+@@ -1933,6 +1933,7 @@ static cudaError_t ggml_cuda_copy2d_across_devices(
+     size_t width, size_t height, cudaStream_t dst_stream, cudaStream_t src_stream) {
+ 
+     const auto & info = ggml_cuda_info();
++#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)  // 3D-peer copy types unmapped by ggml's HIP/MUSA shim; use staging fallback below
+     if (info.peer_access[src_device][dst_device]) {
+         cudaMemcpy3DPeerParms p = {};
+         p.dstDevice = dst_device;
+@@ -1942,6 +1943,7 @@ static cudaError_t ggml_cuda_copy2d_across_devices(
+         p.extent = make_cudaExtent(width, height, 1);
+         return cudaMemcpy3DPeerAsync(&p, dst_stream);
+     }
++#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+ 
+     // Fallback: stage all rows through a single contiguous pinned buffer
+     int prev_device = ggml_cuda_get_device();
+@@ -5714,7 +5716,7 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
+     ggml_cuda_set_device(dev_ctx->device);
+ 
+     cudaEvent_t event;
+-    CUDA_CHECK(cudaEventCreate(&event));
++    CUDA_CHECK(cudaEventCreateWithFlags(&event, cudaEventDisableTiming));
+ 
+     return new ggml_backend_event {
+         /* .device  = */ dev,

backend/go/crispasr/CMakeLists.txt

@@ -14,7 +14,7 @@ target_include_directories(gocrispasr PRIVATE
 # whisper. crispasr is the referencer; the backend static libs supply the
 # per-architecture symbols; ggml is the math/runtime base.
 target_link_libraries(gocrispasr PRIVATE
-    crispasr
+    crispasr-lib
     parakeet canary canary_ctc cohere granite_speech granite_nle
     voxtral voxtral4b qwen3_asr qwen3_tts orpheus chatterbox indextts
     kokoro voxcpm2_tts m2m100 t5_translate wav2vec2-ggml vibevoice

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=13d54e110e1538e0f0bc3af0680b9ab246cfb48d
+CRISPASR_VERSION?=d745bda4386ae0f9d1d2f23fff8ec95d76428221
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/crispasr/gocrispasr.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/go-audio/audio"
 	"github.com/go-audio/wav"
+	gguf "github.com/gpustack/gguf-parser-go"
 	"github.com/mudler/LocalAI/pkg/grpc/base"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/utils"
@@ -37,6 +38,39 @@ var (
 
 type CrispASR struct {
 	base.SingleThread
+	// sampleRate is the output rate (Hz) of the loaded TTS engine's PCM, used to
+	// write a correct WAV header. Most CrispASR TTS backends emit 24 kHz, but
+	// piper returns its model's native rate (16 kHz for x_low/low voices,
+	// 22.05 kHz for medium/high), so it is read from the GGUF metadata at Load.
+	sampleRate int
+}
+
+// defaultTTSSampleRate is the output rate assumed for CrispASR TTS engines that
+// don't advertise one in GGUF metadata (vibevoice/orpheus/chatterbox/qwen3-tts
+// all emit 24 kHz). piper is the exception and carries piper.sample_rate.
+const defaultTTSSampleRate = 24000
+
+// piperSampleRate reads the piper.sample_rate metadata key from a GGUF model.
+// CrispASR's piper backend returns PCM at the model's native rate without
+// resampling, so the WAV header must match it. Returns ok=false for non-piper
+// models (key absent) or an unreadable file, letting the caller fall back to
+// defaultTTSSampleRate.
+func piperSampleRate(modelPath string) (int, bool) {
+	// Only scalar architecture keys are read, so skip the large array metadata
+	// (phoneme map) and mmap the header - same rationale as pkg/vram's reader.
+	f, err := gguf.ParseGGUFFile(modelPath, gguf.UseMMap(), gguf.SkipLargeMetadata())
+	if err != nil {
+		return 0, false
+	}
+	kv, ok := f.Header.MetadataKV.Get("piper.sample_rate")
+	if !ok || kv.ValueType != gguf.GGUFMetadataValueTypeUint32 {
+		return 0, false
+	}
+	rate := int(kv.ValueUint32())
+	if rate <= 0 {
+		return 0, false
+	}
+	return rate, true
 }
 
 // splitOption splits a "prefix:value" model option into its key and value,
@@ -103,6 +137,14 @@ func (w *CrispASR) Load(opts *pb.ModelOptions) error {
 		return fmt.Errorf("Failed to load CrispASR transcription model")
 	}
 
+	// Determine the TTS output sample rate for the WAV header. piper voices
+	// carry their native rate in GGUF metadata and CrispASR does not resample;
+	// every other engine emits the 24 kHz default.
+	w.sampleRate = defaultTTSSampleRate
+	if rate, ok := piperSampleRate(opts.ModelFile); ok {
+		w.sampleRate = rate
+	}
+
 	// Load the companion file (codec/tokenizer/s3gen) after the session is open.
 	// rc==0 means success or "not applicable" for the active backend; only a
 	// negative code is fatal.
@@ -390,7 +432,7 @@ func (w *CrispASR) synthesize(text string) ([]float32, error) {
 	}
 	defer CppTTSFree(ptr)
 	src := unsafe.Slice((*float32)(unsafe.Pointer(ptr)), int(n)) //nolint:govet // ptr addresses C-allocated PCM returned across the purego boundary; copied out immediately below, before tts_free.
-	out := make([]float32, int(n)) // copy out of C memory before free
+	out := make([]float32, int(n))                               // copy out of C memory before free
 	copy(out, src)
 	return out, nil
 }
@@ -417,7 +459,7 @@ func (w *CrispASR) TTS(req *pb.TTSRequest) error {
 	if err != nil {
 		return err
 	}
-	return writeWAV24k(req.Dst, pcm)
+	return writeWAV(req.Dst, pcm, w.sampleRate)
 }
 
 // TTSStream is the streaming counterpart to TTS. CrispASR has no progressive
@@ -447,7 +489,7 @@ func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error {
 	}
 	defer func() { _ = os.Remove(dst) }()
 
-	if err := writeWAV24k(dst, pcm); err != nil {
+	if err := writeWAV(dst, pcm, w.sampleRate); err != nil {
 		return err
 	}
 
@@ -459,14 +501,14 @@ func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error {
 	return nil
 }
 
-// writeWAV24k writes pcm as a 24000 Hz, mono, 16-bit PCM WAV at dst.
-func writeWAV24k(dst string, pcm []float32) error {
+// writeWAV writes pcm as a sampleRate Hz, mono, 16-bit PCM WAV at dst.
+func writeWAV(dst string, pcm []float32, sampleRate int) error {
 	f, err := os.Create(dst)
 	if err != nil {
 		return fmt.Errorf("crispasr: create %q: %w", dst, err)
 	}
 
-	enc := wav.NewEncoder(f, 24000, 16, 1, 1)
+	enc := wav.NewEncoder(f, sampleRate, 16, 1, 1)
 	ints := make([]int, len(pcm))
 	for i, s := range pcm {
 		if s > 1 {
@@ -477,7 +519,7 @@ func writeWAV24k(dst string, pcm []float32) error {
 		ints[i] = int(s * 32767)
 	}
 	buf := &audio.IntBuffer{
-		Format:         &audio.Format{NumChannels: 1, SampleRate: 24000},
+		Format:         &audio.Format{NumChannels: 1, SampleRate: sampleRate},
 		Data:           ints,
 		SourceBitDepth: 16,
 	}

backend/go/crispasr/gocrispasr_samplerate_test.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+
+	"github.com/go-audio/wav"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// GGUF metadata value type tags (subset) from the GGUF spec.
+const (
+	ggufTypeUint32 uint32 = 4
+	ggufTypeString uint32 = 8
+)
+
+type ggufKV struct {
+	key   string
+	vtype uint32
+	val   any
+}
+
+// writeMinimalGGUF emits a valid, tensor-less GGUF file carrying only the given
+// metadata key-values. Enough for the header-only parse path piperSampleRate
+// uses; avoids pulling a real multi-MB voice into the test.
+func writeMinimalGGUF(path string, kvs []ggufKV) error {
+	var b bytes.Buffer
+	b.WriteString("GGUF")                                // magic
+	_ = binary.Write(&b, binary.LittleEndian, uint32(3)) // version
+	_ = binary.Write(&b, binary.LittleEndian, uint64(0)) // tensor count
+	_ = binary.Write(&b, binary.LittleEndian, uint64(len(kvs)))
+	for _, kv := range kvs {
+		_ = binary.Write(&b, binary.LittleEndian, uint64(len(kv.key)))
+		b.WriteString(kv.key)
+		_ = binary.Write(&b, binary.LittleEndian, kv.vtype)
+		switch v := kv.val.(type) {
+		case uint32:
+			_ = binary.Write(&b, binary.LittleEndian, v)
+		case string:
+			_ = binary.Write(&b, binary.LittleEndian, uint64(len(v)))
+			b.WriteString(v)
+		}
+	}
+	return os.WriteFile(path, b.Bytes(), 0o644)
+}
+
+// wavSampleRate decodes the WAV header at path and returns its sample rate.
+func wavSampleRate(path string) (int, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = f.Close() }()
+	dec := wav.NewDecoder(f)
+	dec.ReadInfo()
+	return int(dec.SampleRate), nil
+}
+
+var _ = Describe("piper sample rate", func() {
+	Context("piperSampleRate", func() {
+		It("reads piper.sample_rate from a piper GGUF (medium = 22050)", func() {
+			p := filepath.Join(GinkgoT().TempDir(), "voice.gguf")
+			Expect(writeMinimalGGUF(p, []ggufKV{
+				{key: "general.architecture", vtype: ggufTypeString, val: "piper"},
+				{key: "piper.sample_rate", vtype: ggufTypeUint32, val: uint32(22050)},
+			})).To(Succeed())
+
+			rate, ok := piperSampleRate(p)
+			Expect(ok).To(BeTrue(), "piper.sample_rate should be found")
+			Expect(rate).To(Equal(22050))
+		})
+
+		It("reads the low-quality rate (16000)", func() {
+			p := filepath.Join(GinkgoT().TempDir(), "voice.gguf")
+			Expect(writeMinimalGGUF(p, []ggufKV{
+				{key: "piper.sample_rate", vtype: ggufTypeUint32, val: uint32(16000)},
+			})).To(Succeed())
+
+			rate, ok := piperSampleRate(p)
+			Expect(ok).To(BeTrue())
+			Expect(rate).To(Equal(16000))
+		})
+
+		It("returns ok=false for a non-piper GGUF (no piper.sample_rate key)", func() {
+			p := filepath.Join(GinkgoT().TempDir(), "other.gguf")
+			Expect(writeMinimalGGUF(p, []ggufKV{
+				{key: "general.architecture", vtype: ggufTypeString, val: "vibevoice"},
+			})).To(Succeed())
+
+			_, ok := piperSampleRate(p)
+			Expect(ok).To(BeFalse())
+		})
+
+		It("returns ok=false for an unreadable/non-GGUF file", func() {
+			p := filepath.Join(GinkgoT().TempDir(), "garbage.gguf")
+			Expect(os.WriteFile(p, []byte("not a gguf"), 0o644)).To(Succeed())
+
+			_, ok := piperSampleRate(p)
+			Expect(ok).To(BeFalse())
+		})
+	})
+
+	// End-to-end through the built .so. Gated on CRISPASR_PIPER_MODEL_PATH (a
+	// real piper voice GGUF) like the other model-backed specs; never runs in
+	// default CI. Proves CrispASR's piper backend output rate flows into the
+	// WAV header instead of the hardcoded 24 kHz default.
+	Context("piper TTS end-to-end", func() {
+		It("writes the WAV at the model's native piper.sample_rate", func() {
+			model := os.Getenv("CRISPASR_PIPER_MODEL_PATH")
+			if model == "" {
+				Skip("set CRISPASR_PIPER_MODEL_PATH to run the piper e2e spec")
+			}
+			ensureLibLoaded()
+
+			expected, ok := piperSampleRate(model)
+			Expect(ok).To(BeTrue(), "model should carry piper.sample_rate metadata")
+
+			w := &CrispASR{}
+			Expect(w.Load(&pb.ModelOptions{
+				ModelFile: model,
+				Options:   []string{"backend:piper"},
+				Threads:   4,
+			})).To(Succeed())
+
+			dst := filepath.Join(GinkgoT().TempDir(), "piper.wav")
+			Expect(w.TTS(&pb.TTSRequest{Text: "Hello from CrispASR piper.", Dst: dst})).To(Succeed())
+
+			info, err := os.Stat(dst)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(info.Size()).To(BeNumerically(">", 1024), "expected a non-trivial WAV")
+
+			rate, err := wavSampleRate(dst)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(rate).To(Equal(expected),
+				"WAV header rate must equal the model's native piper.sample_rate, not the 24k default")
+		})
+	})
+
+	Context("writeWAV", func() {
+		It("writes the WAV header at the given sample rate (22050 for piper, not the 24k default)", func() {
+			dst := filepath.Join(GinkgoT().TempDir(), "out.wav")
+			pcm := make([]float32, 220) // 10 ms of silence is enough for a header
+			Expect(writeWAV(dst, pcm, 22050)).To(Succeed())
+
+			rate, err := wavSampleRate(dst)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(rate).To(Equal(22050))
+		})
+
+		It("writes a 16000 Hz header for low-quality piper voices", func() {
+			dst := filepath.Join(GinkgoT().TempDir(), "out.wav")
+			pcm := make([]float32, 160)
+			Expect(writeWAV(dst, pcm, 16000)).To(Succeed())
+
+			rate, err := wavSampleRate(dst)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(rate).To(Equal(16000))
+		})
+	})
+})

backend/go/crispasr/package.sh

@@ -51,6 +51,32 @@ else
     exit 1
 fi
 
+# Bundle espeak-ng (+ its libpcaudio/libsonic runtime deps) and its voice data so
+# the piper TTS backend can phonemize non-English text. CrispASR dlopens
+# libespeak-ng.so.1 at runtime (the MIT-clean path); the dlopen succeeds loading
+# libespeak-ng but FAILS if libpcaudio/libsonic are absent, so all three .so are
+# required. run.sh points CRISPASR_ESPEAK_DATA_PATH at the bundled data dir.
+# Best-effort: only copied when present, so a local dev build without espeak-ng
+# installed still packages the rest (English voices keep working).
+ESPEAK_LIBDIR=""
+for d in /usr/lib/x86_64-linux-gnu /usr/lib/aarch64-linux-gnu; do
+    if [ -f "$d/libespeak-ng.so.1" ]; then
+        ESPEAK_LIBDIR="$d"
+        break
+    fi
+done
+if [ -n "$ESPEAK_LIBDIR" ]; then
+    echo "Bundling espeak-ng from $ESPEAK_LIBDIR ..."
+    cp -arfLv "$ESPEAK_LIBDIR/libespeak-ng.so.1" $CURDIR/package/lib/
+    cp -arfLv "$ESPEAK_LIBDIR/libpcaudio.so.0" $CURDIR/package/lib/
+    cp -arfLv "$ESPEAK_LIBDIR/libsonic.so.0" $CURDIR/package/lib/
+    if [ -d "$ESPEAK_LIBDIR/espeak-ng-data" ]; then
+        cp -arfLv "$ESPEAK_LIBDIR/espeak-ng-data" $CURDIR/package/
+    fi
+else
+    echo "espeak-ng not found; non-English piper voices will not phonemize"
+fi
+
 # Package GPU libraries based on BUILD_TYPE
 # The GPU library packaging script will detect BUILD_TYPE and copy appropriate GPU libraries
 GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"

backend/go/crispasr/run.sh

@@ -41,6 +41,11 @@ fi
 export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
 export CRISPASR_LIBRARY=$LIBRARY
 
+# Point piper's espeak-ng phonemizer at the bundled voice data. The variable
+# names the directory CONTAINING espeak-ng-data (package.sh drops it next to
+# this script). Harmless when espeak-ng wasn't bundled.
+export CRISPASR_ESPEAK_DATA_PATH=$CURDIR
+
 # If there is a lib/ld.so, use it
 if [ -f $CURDIR/lib/ld.so ]; then
 	echo "Using lib/ld.so"

backend/go/locate-anything-cpp/.gitignore

@@ -0,0 +1,7 @@
+sources/
+build*/
+package/
+liblocateanythingcpp*.so
+locate-anything-cpp
+test-models/
+test-data/

backend/go/locate-anything-cpp/CMakeLists.txt

@@ -0,0 +1,57 @@
+cmake_minimum_required(VERSION 3.18)
+project(liblocateanythingcpp LANGUAGES C CXX)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Static-link ggml + locate_anything so the resulting .so has no runtime
+# dependency on extra ggml/locate_anything shared libraries — only on
+# libc/libstdc++/libgomp, which the LocalAI package step bundles into the
+# docker image.
+set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries" FORCE)
+
+# locate-anything.cpp build switches: skip CLI/tests, keep static lib.
+set(LA_BUILD_CLI OFF CACHE BOOL "Disable locate-anything CLI" FORCE)
+set(LA_BUILD_TESTS OFF CACHE BOOL "Disable locate-anything tests" FORCE)
+set(LA_SHARED OFF CACHE BOOL "Build locate_anything as static lib" FORCE)
+
+# Unlike rt-detr.cpp, locate-anything.cpp ships no in-tree ggml patches, so
+# there is no apply_ggml_patches.sh hook to shim here.
+add_subdirectory(./sources/locate-anything.cpp)
+
+# locate-anything.cpp's top-level CMakeLists points its own target's include
+# dirs at ${CMAKE_SOURCE_DIR}/{include,src,third_party,...}. CMAKE_SOURCE_DIR
+# is the *top-level* source dir of the whole CMake tree, so when we pull it in
+# via add_subdirectory it resolves to OUR directory, not theirs, and the
+# locate_anything target fails to find its own headers (la_capi.h, stb_image.h,
+# la_gguf_keys.h). Re-add the correct, subdir-relative include paths to the
+# already-defined target so it compiles regardless of where it's nested.
+set(LA_SRC ${CMAKE_CURRENT_SOURCE_DIR}/sources/locate-anything.cpp)
+target_include_directories(locate_anything PRIVATE
+    ${LA_SRC}/include
+    ${LA_SRC}/src
+    ${LA_SRC}/third_party
+    ${LA_SRC}/third_party/stb)
+
+# locate-anything.cpp's C-API symbols already live inside liblocate_anything
+# (src/la_capi.cpp is compiled into the lib). We re-export them via a MODULE
+# library that links locate_anything so the symbols are visible at dlopen time.
+add_library(locateanythingcpp MODULE
+    sources/locate-anything.cpp/src/la_capi.cpp)
+
+target_include_directories(locateanythingcpp PRIVATE
+    sources/locate-anything.cpp/include
+    sources/locate-anything.cpp/src
+    sources/locate-anything.cpp/third_party
+    sources/locate-anything.cpp/third_party/stb
+)
+
+target_link_libraries(locateanythingcpp PRIVATE locate_anything ggml)
+
+if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
+    target_link_libraries(locateanythingcpp PRIVATE stdc++fs)
+endif()
+
+set_property(TARGET locateanythingcpp PROPERTY CXX_STANDARD 17)
+set_target_properties(locateanythingcpp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

backend/go/locate-anything-cpp/Makefile

@@ -0,0 +1,134 @@
+CMAKE_ARGS?=
+BUILD_TYPE?=
+NATIVE?=false
+
+GOCMD?=go
+GO_TAGS?=
+JOBS?=$(shell nproc --ignore=1)
+
+# locate-anything.cpp. Pin to a specific commit for a stable build; leaving
+# this on `master` always picks up the latest C-API surface (incl. the
+# per-detection accessor functions used by golocateanythingcpp.go).
+LOCATEANYTHING_REPO?=https://github.com/mudler/locate-anything.cpp.git
+LOCATEANYTHING_VERSION?=60e450945476d5e97e0754a8c0e71a9ea81690e0
+
+ifeq ($(NATIVE),false)
+	CMAKE_ARGS+=-DGGML_NATIVE=OFF
+endif
+
+# Forward LocalAI's BUILD_TYPE to the matching ggml backend switch.
+ifeq ($(BUILD_TYPE),cublas)
+	CMAKE_ARGS+=-DGGML_CUDA=ON -DLA_GGML_CUDA=ON
+else ifeq ($(BUILD_TYPE),openblas)
+	CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
+else ifeq ($(BUILD_TYPE),clblas)
+	CMAKE_ARGS+=-DGGML_CLBLAST=ON
+else ifeq ($(BUILD_TYPE),hipblas)
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
+else ifeq ($(BUILD_TYPE),vulkan)
+	CMAKE_ARGS+=-DGGML_VULKAN=ON -DLA_GGML_VULKAN=ON
+else ifeq ($(OS),Darwin)
+	ifneq ($(BUILD_TYPE),metal)
+		CMAKE_ARGS+=-DGGML_METAL=OFF
+	else
+		CMAKE_ARGS+=-DGGML_METAL=ON
+		CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON
+		CMAKE_ARGS+=-DLA_GGML_METAL=ON
+	endif
+endif
+
+ifeq ($(BUILD_TYPE),sycl_f16)
+	CMAKE_ARGS+=-DGGML_SYCL=ON \
+		-DCMAKE_C_COMPILER=icx \
+		-DCMAKE_CXX_COMPILER=icpx \
+		-DGGML_SYCL_F16=ON
+endif
+
+ifeq ($(BUILD_TYPE),sycl_f32)
+	CMAKE_ARGS+=-DGGML_SYCL=ON \
+		-DCMAKE_C_COMPILER=icx \
+		-DCMAKE_CXX_COMPILER=icpx
+endif
+
+sources/locate-anything.cpp:
+	mkdir -p sources && \
+	git clone --recursive $(LOCATEANYTHING_REPO) sources/locate-anything.cpp && \
+	cd sources/locate-anything.cpp && \
+	git checkout $(LOCATEANYTHING_VERSION) && \
+	git submodule update --init --recursive --depth 1 --single-branch
+
+# Detect OS
+UNAME_S := $(shell uname -s)
+
+# Only build CPU variants on Linux
+ifeq ($(UNAME_S),Linux)
+	VARIANT_TARGETS = liblocateanythingcpp-avx.so liblocateanythingcpp-avx2.so liblocateanythingcpp-avx512.so liblocateanythingcpp-fallback.so
+else
+	# On non-Linux (e.g., Darwin), build only fallback variant
+	VARIANT_TARGETS = liblocateanythingcpp-fallback.so
+endif
+
+locate-anything-cpp: main.go golocateanythingcpp.go $(VARIANT_TARGETS)
+	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o locate-anything-cpp ./
+
+package: locate-anything-cpp
+	bash package.sh
+
+build: package
+
+clean: purge
+	rm -rf liblocateanythingcpp*.so locate-anything-cpp package sources
+
+purge:
+	rm -rf build*
+
+# Build all variants (Linux only)
+ifeq ($(UNAME_S),Linux)
+liblocateanythingcpp-avx.so: sources/locate-anything.cpp
+	rm -rfv build-$@
+	$(info ${GREEN}I locate-anything-cpp build info:avx${RESET})
+	SO_TARGET=$@ CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) liblocateanythingcpp-custom
+	rm -rfv build-$@
+
+liblocateanythingcpp-avx2.so: sources/locate-anything.cpp
+	rm -rfv build-$@
+	$(info ${GREEN}I locate-anything-cpp build info:avx2${RESET})
+	SO_TARGET=$@ CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) liblocateanythingcpp-custom
+	rm -rfv build-$@
+
+liblocateanythingcpp-avx512.so: sources/locate-anything.cpp
+	rm -rfv build-$@
+	$(info ${GREEN}I locate-anything-cpp build info:avx512${RESET})
+	SO_TARGET=$@ CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) liblocateanythingcpp-custom
+	rm -rfv build-$@
+endif
+
+# Build fallback variant (all platforms)
+liblocateanythingcpp-fallback.so: sources/locate-anything.cpp
+	rm -rfv build-$@
+	$(info ${GREEN}I locate-anything-cpp build info:fallback${RESET})
+	SO_TARGET=$@ CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) liblocateanythingcpp-custom
+	rm -rfv build-$@
+
+liblocateanythingcpp-custom: CMakeLists.txt
+	mkdir -p build-$(SO_TARGET) && \
+	cd build-$(SO_TARGET) && \
+	cmake .. $(CMAKE_ARGS) && \
+	cmake --build . --config Release -j$(JOBS) && \
+	cd .. && \
+	mv build-$(SO_TARGET)/liblocateanythingcpp.so ./$(SO_TARGET)
+
+all: locate-anything-cpp package
+
+# `test` is invoked by the top-level Makefile's `test-extra` target. It builds
+# the backend binary + the fallback shared library (needed for dlopen at
+# runtime), then runs test.sh which downloads the q8_0 GGUF + COCO image and
+# exercises the gRPC Load/Detect wire path via the Go smoke test in
+# main_test.go.
+test: locate-anything-cpp liblocateanythingcpp-fallback.so
+	bash test.sh

backend/go/locate-anything-cpp/golocateanythingcpp.go

@@ -0,0 +1,174 @@
+package main
+
+// golocateanythingcpp.go - gRPC handlers (Load, Detect) for the
+// locate-anything-cpp backend.
+//
+// Embeds base.SingleThread to default unimplemented RPCs to "not supported"
+// while we only implement open-vocabulary object detection (Detect).
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+	"unsafe"
+
+	"github.com/mudler/LocalAI/pkg/grpc/base"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+)
+
+// la_ctx* is an opaque handle. la_capi_load returns it directly (0 == failure),
+// unlike rfdetr's out-parameter convention.
+var (
+	// la_capi_load(const char* gguf_path, int n_threads) -> la_ctx* (0 = fail)
+	CapiLoad func(gguf string, nThreads int32) uintptr
+	// la_capi_free(la_ctx* ctx)
+	CapiFree func(handle uintptr)
+	// la_capi_locate_path(ctx, image_path, prompt, mode) -> char* json (0 = err)
+	CapiLocatePath func(handle uintptr, imagePath string, prompt string, mode int32) uintptr
+	// la_capi_locate_buffer(ctx, bytes, len, prompt, mode) -> char* json (0 = err)
+	CapiLocateBuffer func(handle uintptr, bytes uintptr, length uintptr, prompt string, mode int32) uintptr
+	// la_capi_get_n_detections(ctx) -> int
+	CapiGetNDetections func(handle uintptr) int32
+	// la_capi_get_detection_box(ctx, i, out_xyxy[4]) -> int (0 on success)
+	CapiGetDetectionBox func(handle uintptr, i int32, outXYXY uintptr) int32
+	// la_capi_get_detection_label(ctx, i, buf, buf_size) -> int (required size incl NUL; two-call sizing)
+	CapiGetDetectionLabel func(handle uintptr, i int32, buf uintptr, bufSize int32) int32
+	// la_capi_free_string(char* s)
+	CapiFreeString func(s uintptr)
+	// la_capi_last_error(ctx) -> const char* (owned by ctx, "" if none / null ctx).
+	// purego marshals the returned C string into a Go string (a copy), so we
+	// never free it and avoid raw pointer arithmetic.
+	CapiLastError func(handle uintptr) string
+)
+
+type LocateAnythingCpp struct {
+	base.SingleThread
+	handle uintptr
+}
+
+// Load loads the GGUF model at opts.ModelFile (joined with opts.ModelPath if
+// relative) and stores the la_ctx handle for later Detect calls.
+func (r *LocateAnythingCpp) Load(opts *pb.ModelOptions) error {
+	modelFile := opts.ModelFile
+	if modelFile == "" {
+		modelFile = opts.Model
+	}
+	if modelFile == "" {
+		return fmt.Errorf("locate-anything-cpp: ModelFile is empty")
+	}
+
+	var modelPath string
+	if filepath.IsAbs(modelFile) {
+		modelPath = modelFile
+	} else {
+		modelPath = filepath.Join(opts.ModelPath, modelFile)
+	}
+
+	if _, err := os.Stat(modelPath); err != nil {
+		return fmt.Errorf("locate-anything-cpp: model file not found: %s: %w", modelPath, err)
+	}
+
+	threads := opts.Threads
+	if threads <= 0 {
+		threads = 4
+	}
+
+	// Release previous model if any (re-Load).
+	if r.handle != 0 {
+		CapiFree(r.handle)
+		r.handle = 0
+	}
+
+	h := CapiLoad(modelPath, threads)
+	if h == 0 {
+		// la_capi_last_error needs a ctx; on a failed load we have none (it
+		// returns "" for a null ctx), so the text is best-effort. Surface it
+		// when present.
+		if msg := CapiLastError(0); msg != "" {
+			return fmt.Errorf("locate-anything-cpp: la_capi_load failed for %s: %s", modelPath, msg)
+		}
+		return fmt.Errorf("locate-anything-cpp: la_capi_load failed for %s", modelPath)
+	}
+	r.handle = h
+	return nil
+}
+
+// Detect runs open-vocabulary detection on the base64-encoded image in opts.Src
+// using the required text prompt in opts.Prompt, returning one pb.Detection per
+// located object with its predicted label as ClassName.
+func (r *LocateAnythingCpp) Detect(opts *pb.DetectOptions) (pb.DetectResponse, error) {
+	if r.handle == 0 {
+		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: model not loaded")
+	}
+
+	// Open-vocabulary detection is prompt-driven; without a prompt there is
+	// nothing to locate.
+	prompt := opts.Prompt
+	if prompt == "" {
+		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: a text prompt is required (open-vocabulary detection)")
+	}
+
+	// Decode base64 image and write to temp file.
+	imgData, err := base64.StdEncoding.DecodeString(opts.Src)
+	if err != nil {
+		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to decode base64 image: %w", err)
+	}
+
+	tmpFile, err := os.CreateTemp("", "locate-anything-*.img")
+	if err != nil {
+		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to create temp file: %w", err)
+	}
+	defer func() { _ = os.Remove(tmpFile.Name()) }()
+
+	if _, err := tmpFile.Write(imgData); err != nil {
+		_ = tmpFile.Close()
+		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to write temp file: %w", err)
+	}
+	if err := tmpFile.Close(); err != nil {
+		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to close temp file: %w", err)
+	}
+
+	// mode 0 = hybrid (Parallel Box Decoding). The JSON return value is unused:
+	// structured detections are read via the accessor functions. Still must
+	// free the returned string.
+	jsonPtr := CapiLocatePath(r.handle, tmpFile.Name(), prompt, 0)
+	if jsonPtr != 0 {
+		CapiFreeString(jsonPtr)
+	}
+
+	n := CapiGetNDetections(r.handle)
+	if n < 0 {
+		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: invalid n_detections=%d", n)
+	}
+
+	detections := make([]*pb.Detection, 0, n)
+	for i := int32(0); i < n; i++ {
+		var xyxy [4]float32 // x1, y1, x2, y2
+		if CapiGetDetectionBox(r.handle, i, uintptr(unsafe.Pointer(&xyxy[0]))) != 0 {
+			continue
+		}
+
+		// Two-call sizing for the label string.
+		label := ""
+		need := CapiGetDetectionLabel(r.handle, i, 0, 0)
+		if need > 0 {
+			buf := make([]byte, need)
+			CapiGetDetectionLabel(r.handle, i, uintptr(unsafe.Pointer(&buf[0])), need)
+			label = string(buf[:need-1])
+		}
+
+		detections = append(detections, &pb.Detection{
+			X:          xyxy[0],
+			Y:          xyxy[1],
+			Width:      xyxy[2] - xyxy[0],
+			Height:     xyxy[3] - xyxy[1],
+			Confidence: 1.0,
+			ClassName:  label,
+		})
+	}
+
+	return pb.DetectResponse{
+		Detections: detections,
+	}, nil
+}

backend/go/locate-anything-cpp/main.go

@@ -0,0 +1,59 @@
+package main
+
+// main.go - entry point for the locate-anything-cpp gRPC backend.
+//
+// Dlopens liblocateanythingcpp-<variant>.so via purego at the path in
+// LOCATEANYTHING_LIBRARY (set by run.sh based on /proc/cpuinfo), registers
+// the la_capi_* C ABI symbols, then starts the gRPC server.
+
+import (
+	"flag"
+	"os"
+
+	"github.com/ebitengine/purego"
+	grpc "github.com/mudler/LocalAI/pkg/grpc"
+)
+
+var (
+	addr = flag.String("addr", "localhost:50051", "the address to connect to")
+)
+
+type LibFuncs struct {
+	FuncPtr any
+	Name    string
+}
+
+func main() {
+	// Get library name from environment variable, default to fallback
+	libName := os.Getenv("LOCATEANYTHING_LIBRARY")
+	if libName == "" {
+		libName = "./liblocateanythingcpp-fallback.so"
+	}
+
+	lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL)
+	if err != nil {
+		panic(err)
+	}
+
+	libFuncs := []LibFuncs{
+		{&CapiLoad, "la_capi_load"},
+		{&CapiFree, "la_capi_free"},
+		{&CapiLocatePath, "la_capi_locate_path"},
+		{&CapiLocateBuffer, "la_capi_locate_buffer"},
+		{&CapiGetNDetections, "la_capi_get_n_detections"},
+		{&CapiGetDetectionBox, "la_capi_get_detection_box"},
+		{&CapiGetDetectionLabel, "la_capi_get_detection_label"},
+		{&CapiFreeString, "la_capi_free_string"},
+		{&CapiLastError, "la_capi_last_error"},
+	}
+
+	for _, lf := range libFuncs {
+		purego.RegisterLibFunc(lf.FuncPtr, lib, lf.Name)
+	}
+
+	flag.Parse()
+
+	if err := grpc.StartServer(*addr, &LocateAnythingCpp{}); err != nil {
+		panic(err)
+	}
+}

backend/go/locate-anything-cpp/main_test.go

@@ -0,0 +1,176 @@
+package main
+
+// main_test.go - end-to-end smoke test for the locate-anything-cpp gRPC backend.
+//
+// Spawns the compiled locate-anything-cpp binary on a free local port, dials it
+// via gRPC, and exercises LoadModel + Detect against the test fixtures
+// downloaded by test.sh: the q8_0 GGUF of nvidia/LocateAnything-3B and a real
+// COCO image with people + cars. Asserts that open-vocabulary detection driven
+// by a text prompt returns at least one detection, each carrying a non-empty
+// class name and a bounding box of non-zero size.
+//
+// The spec Skip()s cleanly if its fixtures (the ~6.3 GB model, the test image,
+// the built binary, or the fallback .so) are missing, so the test target stays
+// usable on a fresh checkout / on CI runners where the large model hasn't been
+// downloaded.
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func TestDetect(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "locate-anything-cpp backend smoke suite")
+}
+
+// freePort grabs an ephemeral TCP port and immediately releases it so the
+// spawned backend can bind to it. There is a tiny TOCTOU window here but in
+// practice it's adequate for a smoke test on a quiet runner.
+func freePort() int {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	Expect(err).ToNot(HaveOccurred(), "freePort listen")
+	port := l.Addr().(*net.TCPAddr).Port
+	Expect(l.Close()).To(Succeed())
+	return port
+}
+
+// startBackend spawns the locate-anything-cpp binary on the given port and
+// waits until it accepts TCP connections (up to 10s). It mirrors how main.go
+// resolves the purego library: the LOCATEANYTHING_LIBRARY env var points the
+// dlopen at the freshly built fallback .so, and the la_capi_* symbols are
+// registered there. The returned cleanup func kills the process and reaps it.
+func startBackend(port int) func() {
+	binary, err := filepath.Abs("./locate-anything-cpp")
+	Expect(err).ToNot(HaveOccurred())
+	if _, err := os.Stat(binary); err != nil {
+		Skip(fmt.Sprintf("backend binary not built: %s (run `make locate-anything-cpp` first)", binary))
+	}
+
+	libPath, err := filepath.Abs("./liblocateanythingcpp-fallback.so")
+	Expect(err).ToNot(HaveOccurred())
+	if _, err := os.Stat(libPath); err != nil {
+		Skip(fmt.Sprintf("fallback library not built: %s (run `make liblocateanythingcpp-fallback.so` first)", libPath))
+	}
+
+	addr := fmt.Sprintf("127.0.0.1:%d", port)
+	cmd := exec.Command(binary, "--addr", addr)
+	cmd.Env = append(os.Environ(), "LOCATEANYTHING_LIBRARY="+libPath)
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	Expect(cmd.Start()).To(Succeed())
+
+	cleanup := func() {
+		if cmd.Process != nil {
+			_ = cmd.Process.Kill()
+			_, _ = cmd.Process.Wait()
+		}
+	}
+
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		c, err := net.DialTimeout("tcp", addr, 200*time.Millisecond)
+		if err == nil {
+			_ = c.Close()
+			return cleanup
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	cleanup()
+	Fail(fmt.Sprintf("backend did not become ready on %s within 10s", addr))
+	return func() {}
+}
+
+// loadTestImage reads the COCO test image downloaded by test.sh and returns its
+// base64-encoded content (the wire format accepted by the Detect RPC).
+func loadTestImage() string {
+	imgPath, err := filepath.Abs("test-data/test.jpg")
+	Expect(err).ToNot(HaveOccurred())
+	imgBytes, err := os.ReadFile(imgPath)
+	if err != nil {
+		Skip(fmt.Sprintf("test image not present: %s (run test.sh first)", imgPath))
+	}
+	return base64.StdEncoding.EncodeToString(imgBytes)
+}
+
+// dialBackend opens a gRPC client connection to the spawned backend.
+func dialBackend(port int) (pb.BackendClient, func()) {
+	addr := fmt.Sprintf("127.0.0.1:%d", port)
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	Expect(err).ToNot(HaveOccurred())
+	return pb.NewBackendClient(conn), func() { _ = conn.Close() }
+}
+
+// modelPathOrSkip resolves the model file under ./test-models/ and Skip()s the
+// current spec if it's missing (the ~6.3 GB GGUF is not present on a fresh
+// checkout / on CI runners without the download).
+func modelPathOrSkip(name string) string {
+	modelDir, err := filepath.Abs("test-models")
+	Expect(err).ToNot(HaveOccurred())
+	modelPath := filepath.Join(modelDir, name)
+	if _, err := os.Stat(modelPath); err != nil {
+		Skip(fmt.Sprintf("model not present: %s (run test.sh first)", modelPath))
+	}
+	return modelPath
+}
+
+var _ = Describe("locate-anything-cpp backend", func() {
+	It("runs open-vocabulary detection against a known-good COCO image", func() {
+		modelPath := modelPathOrSkip("locate-anything-q8_0.gguf")
+		imgB64 := loadTestImage()
+
+		port := freePort()
+		cleanup := startBackend(port)
+		defer cleanup()
+
+		client, closeConn := dialBackend(port)
+		defer closeConn()
+
+		// The q8_0 model is ~6.3 GB and hybrid Parallel Box Decoding on CPU is
+		// not cheap, so give LoadModel + Detect a generous deadline.
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
+		defer cancel()
+
+		loadResp, err := client.LoadModel(ctx, &pb.ModelOptions{
+			Model:     "locate-anything-q8_0.gguf",
+			ModelFile: modelPath,
+			Threads:   4,
+		})
+		Expect(err).ToNot(HaveOccurred(), "LoadModel")
+		Expect(loadResp.GetSuccess()).To(BeTrue(), "LoadModel reported failure: %s", loadResp.GetMessage())
+
+		// Open-vocabulary detection is prompt-driven; the prompt names the
+		// classes to locate (people + cars), separated by the </c> control token.
+		detResp, err := client.Detect(ctx, &pb.DetectOptions{
+			Src:    imgB64,
+			Prompt: "Locate all the instances that matches the following description: person</c>car.",
+		})
+		Expect(err).ToNot(HaveOccurred(), "Detect")
+		Expect(detResp.GetDetections()).ToNot(BeEmpty(), "no detections returned on a known-good COCO image")
+
+		_, _ = fmt.Fprintf(GinkgoWriter, "detection OK: %d detections\n", len(detResp.GetDetections()))
+		for i, d := range detResp.GetDetections() {
+			Expect(d.GetClassName()).ToNot(BeEmpty(), "detection %d has empty class_name", i)
+			Expect(d.GetWidth()).To(BeNumerically(">", float32(0)),
+				"detection %d has non-positive width", i)
+			Expect(d.GetHeight()).To(BeNumerically(">", float32(0)),
+				"detection %d has non-positive height", i)
+			_, _ = fmt.Fprintf(GinkgoWriter, "  [%d] %s box=(%.1f,%.1f,%.1fx%.1f)\n",
+				i, d.GetClassName(), d.GetX(), d.GetY(), d.GetWidth(), d.GetHeight())
+		}
+	})
+})

backend/go/locate-anything-cpp/package.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Script to copy the appropriate libraries based on architecture
+
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+REPO_ROOT="${CURDIR}/../../.."
+
+# Create lib directory
+mkdir -p $CURDIR/package/lib
+
+cp -avf $CURDIR/liblocateanythingcpp-*.so $CURDIR/package/
+cp -avf $CURDIR/locate-anything-cpp $CURDIR/package/
+cp -fv $CURDIR/run.sh $CURDIR/package/
+
+# Detect architecture and copy appropriate libraries
+if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
+    # x86_64 architecture
+    echo "Detected x86_64 architecture, copying x86_64 libraries..."
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
+    # ARM64 architecture
+    echo "Detected ARM64 architecture, copying ARM64 libraries..."
+    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+elif [ $(uname -s) = "Darwin" ]; then
+    echo "Detected Darwin"
+else
+    echo "Error: Could not detect architecture"
+    exit 1
+fi
+
+# Package GPU libraries based on BUILD_TYPE
+GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
+if [ -f "$GPU_LIB_SCRIPT" ]; then
+    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
+    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    package_gpu_libs
+fi
+
+echo "Packaging completed successfully"
+ls -liah $CURDIR/package/
+ls -liah $CURDIR/package/lib/

backend/go/locate-anything-cpp/run.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -ex
+
+# Get the absolute current dir where the script is located
+CURDIR=$(dirname "$(realpath $0)")
+
+cd /
+
+echo "CPU info:"
+if [ "$(uname)" != "Darwin" ]; then
+	grep -e "model\sname" /proc/cpuinfo | head -1
+	grep -e "flags" /proc/cpuinfo | head -1
+fi
+
+LIBRARY="$CURDIR/liblocateanythingcpp-fallback.so"
+
+if [ "$(uname)" != "Darwin" ]; then
+	if grep -q -e "\savx\s" /proc/cpuinfo ; then
+		echo "CPU:    AVX    found OK"
+		if [ -e $CURDIR/liblocateanythingcpp-avx.so ]; then
+			LIBRARY="$CURDIR/liblocateanythingcpp-avx.so"
+		fi
+	fi
+
+	if grep -q -e "\savx2\s" /proc/cpuinfo ; then
+		echo "CPU:    AVX2   found OK"
+		if [ -e $CURDIR/liblocateanythingcpp-avx2.so ]; then
+			LIBRARY="$CURDIR/liblocateanythingcpp-avx2.so"
+		fi
+	fi
+
+	# Check avx 512
+	if grep -q -e "\savx512f\s" /proc/cpuinfo ; then
+		echo "CPU:    AVX512F found OK"
+		if [ -e $CURDIR/liblocateanythingcpp-avx512.so ]; then
+			LIBRARY="$CURDIR/liblocateanythingcpp-avx512.so"
+		fi
+	fi
+fi
+
+export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
+export LOCATEANYTHING_LIBRARY=$LIBRARY
+
+# If there is a lib/ld.so, use it
+if [ -f $CURDIR/lib/ld.so ]; then
+	echo "Using lib/ld.so"
+	echo "Using library: $LIBRARY"
+	exec $CURDIR/lib/ld.so $CURDIR/locate-anything-cpp "$@"
+fi
+
+echo "Using library: $LIBRARY"
+exec $CURDIR/locate-anything-cpp "$@"

backend/go/locate-anything-cpp/test.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+
+echo "Running locate-anything-cpp backend tests..."
+
+# Test model from the mudler/locate-anything.cpp-gguf HuggingFace repo. This is
+# the q8_0 quantization of nvidia/LocateAnything-3B (~6.3 GB), so the download
+# is the slow step. It is resumed with `curl -C -` and skipped entirely if the
+# file is already present.
+LOCATEANYTHING_MODEL_DIR="${LOCATEANYTHING_MODEL_DIR:-$CURDIR/test-models}"
+
+LOCATEANYTHING_MODEL_FILE="${LOCATEANYTHING_MODEL_FILE:-locate-anything-q8_0.gguf}"
+LOCATEANYTHING_MODEL_URL="${LOCATEANYTHING_MODEL_URL:-https://huggingface.co/mudler/locate-anything.cpp-gguf/resolve/main/locate-anything-q8_0.gguf}"
+
+mkdir -p "$LOCATEANYTHING_MODEL_DIR"
+
+if [ ! -f "$LOCATEANYTHING_MODEL_DIR/$LOCATEANYTHING_MODEL_FILE" ]; then
+    echo "Downloading locate-anything q8_0 model (~6.3 GB, this is slow)..."
+    # -C - resumes a partial download so an interrupted run doesn't restart from 0.
+    curl -L -C - -o "$LOCATEANYTHING_MODEL_DIR/$LOCATEANYTHING_MODEL_FILE" "$LOCATEANYTHING_MODEL_URL" --progress-bar
+fi
+
+# Use a real COCO test image (people + cars) from the upstream rf-detr.cpp repo
+# (~46 KB). Open-vocabulary detection needs real content to locate, so a
+# synthetic image would trivially yield zero detections.
+TEST_IMAGE_DIR="$CURDIR/test-data"
+TEST_IMAGE_FILE="$TEST_IMAGE_DIR/test.jpg"
+TEST_IMAGE_URL="${TEST_IMAGE_URL:-https://raw.githubusercontent.com/mudler/rf-detr.cpp/main/tests/fixtures/ci/test_image.jpg}"
+
+mkdir -p "$TEST_IMAGE_DIR"
+if [ ! -f "$TEST_IMAGE_FILE" ]; then
+    echo "Downloading COCO test image..."
+    curl -L -o "$TEST_IMAGE_FILE" "$TEST_IMAGE_URL" --progress-bar
+fi
+
+echo "locate-anything-cpp test setup complete."
+echo "  model:      $LOCATEANYTHING_MODEL_DIR/$LOCATEANYTHING_MODEL_FILE"
+echo "  test image: $TEST_IMAGE_FILE"
+
+# Run the Go smoke test: spawns the backend binary on a free port, calls
+# LoadModel + Detect via gRPC against the downloaded GGUF + COCO image.
+echo ""
+echo "Running Go smoke test..."
+cd "$CURDIR"
+go test -v -timeout 30m ./...

backend/go/parakeet-cpp/Makefile

@@ -1,6 +1,6 @@
 # parakeet-cpp backend Makefile.
 #
-# Upstream pin lives below as PARAKEET_VERSION?=b11fe5bca78ad8b342dd559a43d76df3984bb447
+# Upstream pin lives below as PARAKEET_VERSION?=b8012f11e5269126eddb7f4fd02f891a2ccc29b0
 # (.github/bump_deps.sh) can find and update it - matches the
 # whisper.cpp / ds4 / vibevoice-cpp convention.
 #
@@ -15,7 +15,7 @@
 # That's what the L0 smoke test uses. The default target below does the
 # proper clone-at-pin + cmake build so CI doesn't need a side-checkout.
 
-PARAKEET_VERSION?=b11fe5bca78ad8b342dd559a43d76df3984bb447
+PARAKEET_VERSION?=b8012f11e5269126eddb7f4fd02f891a2ccc29b0
 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp
 
 GOCMD?=go
@@ -39,7 +39,10 @@ endif
 # is overwritten back to OFF and the build silently falls back to CPU. Forward the
 # PARAKEET_GGML_* options instead. (openblas is not gated, so -DGGML_BLAS passes through.)
 ifeq ($(BUILD_TYPE),cublas)
-	CMAKE_ARGS+=-DPARAKEET_GGML_CUDA=ON
+	# GGML_CUDA_GRAPHS is OFF by ggml default; enabling it gives a small free
+	# speedup (~1% measured on GB10, never negative) by capturing/replaying the
+	# CUDA graph. Not gated by parakeet.cpp, so it passes straight through to ggml.
+	CMAKE_ARGS+=-DPARAKEET_GGML_CUDA=ON -DGGML_CUDA_GRAPHS=ON
 else ifeq ($(BUILD_TYPE),openblas)
 	CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
 else ifeq ($(BUILD_TYPE),hipblas)

backend/go/parakeet-cpp/batcher.go

@@ -7,8 +7,12 @@ import "time"
 type batchRequest struct {
 	pcm     []float32
 	decoder int32
-	tag     string
-	reply   chan batchReply
+	// language is the per-request target locale ("" means the model default).
+	// parakeet.cpp's batched C-API takes ONE target_lang for the whole batch,
+	// so the dispatcher only coalesces requests that share a language.
+	language string
+	tag      string
+	reply    chan batchReply
 }
 
 // batchReply carries one per-item JSON object string (an element of the C-API's
@@ -43,13 +47,25 @@ func newBatcher(maxSize int, maxWait time.Duration, runBatch func([]*batchReques
 // run is the dispatcher loop: accumulate submitted requests until either maxSize
 // is reached or maxWait elapses since the first queued request, then dispatch.
 // Exits when stop is closed (draining any partially-filled batch first).
+//
+// A batch carries ONE language (parakeet.cpp's batched C-API takes a single
+// target_lang), so a request whose language differs from the batch leader is
+// not coalesced: it is held in carry and becomes the leader of the next batch.
+// carry is therefore never dropped and its caller never deadlocks: every batch
+// (including a lone carry on stop) is dispatched, and runBatch replies to all.
 func (b *batcher) run(stop <-chan struct{}) {
+	var carry *batchRequest
 	for {
 		var first *batchRequest
-		select {
-		case first = <-b.submit:
-		case <-stop:
-			return
+		if carry != nil {
+			// A mismatched request from the previous fill leads this batch.
+			first, carry = carry, nil
+		} else {
+			select {
+			case first = <-b.submit:
+			case <-stop:
+				return
+			}
 		}
 		batch := []*batchRequest{first}
 
@@ -64,12 +80,22 @@ func (b *batcher) run(stop <-chan struct{}) {
 		for len(batch) < b.maxSize {
 			select {
 			case r := <-b.submit:
+				if r.language != first.language {
+					// Different language: carry it to the next batch so this
+					// batch stays single-language, then dispatch what we have.
+					carry = r
+					break fill
+				}
 				batch = append(batch, r)
 			case <-timer.C:
 				break fill
 			case <-stop:
 				timer.Stop()
 				b.runBatch(batch)
+				// Don't strand a carried request's caller on shutdown.
+				if carry != nil {
+					b.runBatch([]*batchRequest{carry})
+				}
 				return
 			}
 		}

backend/go/parakeet-cpp/batcher_test.go

@@ -105,4 +105,60 @@ var _ = Describe("batcher", func() {
 		go func() { <-rep }()
 		Eventually(dispatched, "2s").Should(Receive(Equal(1)))
 	})
+
+	It("never coalesces requests with different languages into one batch", func() {
+		// parakeet.cpp's batched C-API takes ONE target_lang per batch, so the
+		// dispatcher must keep every dispatched batch single-language. Submit a
+		// mix of languages and assert (a) no batch ever carries more than one
+		// distinct language and (b) every submitted request still gets a reply
+		// (the mismatched carry-over is never dropped).
+		var mu sync.Mutex
+		var langsPerBatch [][]string
+		run := func(reqs []*batchRequest) {
+			seen := map[string]struct{}{}
+			var distinct []string
+			for _, r := range reqs {
+				if _, ok := seen[r.language]; !ok {
+					seen[r.language] = struct{}{}
+					distinct = append(distinct, r.language)
+				}
+			}
+			mu.Lock()
+			langsPerBatch = append(langsPerBatch, distinct)
+			mu.Unlock()
+			echoReply(reqs)
+		}
+		// Large window + size so the fill loop stays open across submits and the
+		// language constraint (not the timer) is what splits the batches.
+		b := newBatcher(16, 200*time.Millisecond, run)
+		stop := make(chan struct{})
+		go b.run(stop)
+		defer close(stop)
+
+		langs := []string{"en", "en", "de", "de", "en", "fr", "fr"}
+		const N = 7
+		var wg sync.WaitGroup
+		got := make([]string, N)
+		for i := 0; i < N; i++ {
+			wg.Add(1)
+			go func(i int) {
+				defer wg.Done()
+				rep := make(chan batchReply, 1)
+				b.submit <- &batchRequest{tag: string(rune('a' + i)), language: langs[i], reply: rep}
+				got[i] = (<-rep).json
+			}(i)
+		}
+		wg.Wait()
+
+		mu.Lock()
+		defer mu.Unlock()
+		// Invariant: every dispatched batch is single-language.
+		for _, distinct := range langsPerBatch {
+			Expect(len(distinct)).To(Equal(1), "a batch coalesced more than one language: %v", distinct)
+		}
+		// Liveness: every request got a reply (carry-over never stranded).
+		for i := 0; i < N; i++ {
+			Expect(got[i]).To(Equal(string(rune('a' + i))))
+		}
+	})
 })

backend/go/parakeet-cpp/goparakeetcpp.go

@@ -48,6 +48,13 @@ var (
 	// side reads them as const float*/const int*.
 	CppTranscribePcmBatchJSON func(ctx uintptr, samplesConcat []float32, nSamples []int32, nClips int32, sampleRate int32, decoder int32) uintptr
 
+	// CppTranscribePcmBatchJSONLang is the multilingual variant of the batched
+	// JSON entry point: identical, plus a trailing target_lang. "" (the model
+	// default, "auto") is passed for non-prompt models, which ignore it; an
+	// unknown locale on a prompt model returns 0 and sets last_error. Present
+	// only in newer libparakeet.so; nil falls back to CppTranscribePcmBatchJSON.
+	CppTranscribePcmBatchJSONLang func(ctx uintptr, samplesConcat []float32, nSamples []int32, nClips int32, sampleRate int32, decoder int32, targetLang string) uintptr
+
 	// Cache-aware streaming (RNN-T) entry points. stream_begin returns 0 for
 	// non-streaming models. feed/finalize return a malloc'd char* (uintptr,
 	// freed via CppFreeString); feed writes 1 to *eouOut on an <EOU>/<EOB>.
@@ -55,6 +62,18 @@ var (
 	CppStreamFeed     func(s uintptr, pcm []float32, nSamples int32, eouOut unsafe.Pointer) uintptr
 	CppStreamFinalize func(s uintptr) uintptr
 	CppStreamFree     func(s uintptr)
+
+	// CppStreamBeginLang is the multilingual variant of stream_begin: identical,
+	// plus a trailing target_lang ("" means the model default). Present only in
+	// newer libparakeet.so; nil falls back to CppStreamBegin.
+	CppStreamBeginLang func(ctx uintptr, targetLang string) uintptr
+
+	// Streaming JSON variants (ABI v4): feed/finalize returning a malloc'd char*
+	// JSON document {text,eou,frame_sec,words} (uintptr, freed via CppFreeString)
+	// so streaming segments can carry per-word timestamps. Present only in newer
+	// libparakeet.so; nil falls back to the text-only CppStreamFeed/Finalize path.
+	CppStreamFeedJSON     func(s uintptr, pcm []float32, nSamples int32) uintptr
+	CppStreamFinalizeJSON func(s uintptr) uintptr
 )
 
 // streamChunkSamples is how much 16 kHz mono PCM we hand to stream_feed per
@@ -72,9 +91,30 @@ const streamChunkSamples = 16000
 //
 // "start"/"end"/"t" are seconds; "conf" is confidence in (0,1].
 type transcriptJSON struct {
-	Text   string            `json:"text"`
-	Words  []transcriptWord  `json:"words"`
-	Tokens []transcriptToken `json:"tokens"`
+	Text     string            `json:"text"`
+	FrameSec float64           `json:"frame_sec"`
+	Words    []transcriptWord  `json:"words"`
+	Tokens   []transcriptToken `json:"tokens"`
+}
+
+// streamFeedJSON mirrors the document returned by
+// parakeet_capi_stream_feed_json / parakeet_capi_stream_finalize_json (ABI v5):
+//
+//	{"text":"...","eou":0,"eob":0,"frame_sec":0.080000,
+//	 "words":[{"w":"...","start":0.480,"end":0.640,"conf":0.9100}, ...]}
+//
+// "text" is the newly-finalized text since the last call; "eou" is 1 when an
+// <EOU> (end of utterance) fired this feed and "eob" is 1 when an <EOB>
+// (backchannel) fired. ABI v4 conflated the two into "eou"; v5 split them, so
+// we read both and treat either as an utterance boundary for segmentation.
+// "words" are the words finalized this call with absolute (stream-relative)
+// start/end seconds.
+type streamFeedJSON struct {
+	Text     string           `json:"text"`
+	Eou      int              `json:"eou"`
+	Eob      int              `json:"eob"`
+	FrameSec float64          `json:"frame_sec"`
+	Words    []transcriptWord `json:"words"`
 }
 
 type transcriptWord struct {
@@ -103,6 +143,10 @@ type ParakeetCpp struct {
 	engineMu sync.Mutex // sole guard of the one C engine (dispatcher + streaming)
 	bat      *batcher
 	batStop  chan struct{}
+	// segmentGapFrames is NeMo's segment_gap_threshold in ENCODER FRAMES (model
+	// YAML option, default 0=off). When >0 it adds NeMo's silence-gap split on
+	// top of the punctuation split; converted to seconds via the JSON frame_sec.
+	segmentGapFrames int
 }
 
 // Load is the LocalAI gRPC entry point for LoadModel: it calls
@@ -132,6 +176,11 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error {
 	if maxWaitMs < 0 {
 		maxWaitMs = 0
 	}
+
+	// NeMo's segment_gap_threshold (encoder frames, default 0=off). Off by
+	// default matches NeMo's default (punctuation-only segments); when set it
+	// additionally splits segments on inter-word silence (see transcriptResultFromDoc).
+	p.segmentGapFrames = optInt(opts, "segment_gap_threshold", 0)
 	if CppTranscribePcmBatchJSON != nil {
 		p.batStop = make(chan struct{})
 		p.bat = newBatcher(maxSize, time.Duration(maxWaitMs)*time.Millisecond, p.runBatch)
@@ -187,8 +236,19 @@ func (p *ParakeetCpp) runBatch(reqs []*batchRequest) {
 	if len(reqs) > 0 {
 		dec = reqs[0].decoder
 	}
+	// All requests in a batch share one language (the batcher coalesces only
+	// same-language requests), so any element's language describes the batch.
+	lang := ""
+	if len(reqs) > 0 {
+		lang = reqs[0].language
+	}
 	p.engineMu.Lock()
-	cstr := CppTranscribePcmBatchJSON(p.ctxPtr, concat, nSamples, int32(len(reqs)), 16000, dec)
+	var cstr uintptr
+	if CppTranscribePcmBatchJSONLang != nil {
+		cstr = CppTranscribePcmBatchJSONLang(p.ctxPtr, concat, nSamples, int32(len(reqs)), 16000, dec, lang)
+	} else {
+		cstr = CppTranscribePcmBatchJSON(p.ctxPtr, concat, nSamples, int32(len(reqs)), 16000, dec)
+	}
 	p.engineMu.Unlock()
 	if cstr == 0 {
 		err := fmt.Errorf("parakeet-cpp: batch transcribe failed: %s", CppLastError(p.ctxPtr))
@@ -226,8 +286,9 @@ func (p *ParakeetCpp) runBatch(reqs []*batchRequest) {
 // OpenAI API, whose default is segment-level); token ids always populate
 // Segment.Tokens.
 //
-// translate/diarize/prompt/temperature/language/threads are not applicable to
-// parakeet and are ignored; streaming is handled by AudioTranscriptionStream
+// translate/diarize/prompt/temperature/threads are not applicable to parakeet
+// and are ignored; language is honored on the batched + streaming paths (see
+// opts.GetLanguage() below); streaming is handled by AudioTranscriptionStream
 // (L2).
 func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) {
 	if p.ctxPtr == 0 {
@@ -259,7 +320,7 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip
 		if err := json.Unmarshal([]byte(raw), &doc); err != nil {
 			return pb.TranscriptResult{}, fmt.Errorf("parakeet-cpp: decode transcript json: %w", err)
 		}
-		return transcriptResultFromDoc(doc, opts), nil
+		return transcriptResultFromDoc(doc, opts, p.segmentGapFrames), nil
 	}
 
 	// Batched path: decode to PCM, submit to the batcher, wait for this request's
@@ -271,7 +332,7 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip
 	}
 	rep := make(chan batchReply, 1)
 	select {
-	case p.bat.submit <- &batchRequest{pcm: pcm, decoder: 0, reply: rep}:
+	case p.bat.submit <- &batchRequest{pcm: pcm, decoder: 0, language: opts.GetLanguage(), reply: rep}:
 	case <-ctx.Done():
 		return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled")
 	}
@@ -288,34 +349,172 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip
 	if err := json.Unmarshal([]byte(res.json), &doc); err != nil {
 		return pb.TranscriptResult{}, fmt.Errorf("parakeet-cpp: decode transcript json: %w", err)
 	}
-	return transcriptResultFromDoc(doc, opts), nil
+	return transcriptResultFromDoc(doc, opts, p.segmentGapFrames), nil
 }
 
+// segmentSeparators is NeMo's default segment_seperators (sentence-ending
+// punctuation). Splitting on these matches NeMo's default segment timestamps.
+var segmentSeparators = []rune{'.', '?', '!'}
+
 // transcriptResultFromDoc maps a decoded transcriptJSON to a TranscriptResult,
-// synthesising a single whole-clip segment and attaching word timings only when
-// the caller requested word granularity. Shared by the batched and direct paths.
-func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest) pb.TranscriptResult {
+// grouping words into NeMo-faithful segments (see splitWordsIntoSegments). The
+// optional gapFrames (NeMo's segment_gap_threshold, in encoder FRAMES; 0=off)
+// additionally splits on inter-word silence; it is converted to a seconds gap
+// with the document's frame_sec. Per-segment word timings are attached only when
+// the caller requested word granularity; token ids populate each segment's
+// Tokens by time-window membership. Shared by the batched and direct paths.
+func transcriptResultFromDoc(doc transcriptJSON, opts *pb.TranscriptRequest, gapFrames int) pb.TranscriptResult {
 	text := strings.TrimSpace(doc.Text)
-	words := make([]*pb.TranscriptWord, 0, len(doc.Words))
-	for _, w := range doc.Words {
-		words = append(words, &pb.TranscriptWord{Start: secondsToNanos(w.Start), End: secondsToNanos(w.End), Text: w.W})
+
+	// Frame-unit gap threshold -> seconds (NeMo segment_gap_threshold). 0 = off.
+	gapSeconds := 0.0
+	if gapFrames > 0 {
+		if doc.FrameSec > 0 {
+			gapSeconds = float64(gapFrames) * doc.FrameSec
+		} else {
+			xlog.Warn("parakeet-cpp: segment_gap_threshold set but libparakeet.so " +
+				"did not report frame_sec; falling back to punctuation-only segments")
+		}
 	}
-	tokens := make([]int32, 0, len(doc.Tokens))
-	for _, t := range doc.Tokens {
-		tokens = append(tokens, t.ID)
+
+	groups := splitWordsIntoSegments(doc.Words, segmentSeparators, gapSeconds)
+	if len(groups) == 0 {
+		// No words (edge case): single whole-clip text segment.
+		return pb.TranscriptResult{
+			Text:     text,
+			Segments: []*pb.TranscriptSegment{{Id: 0, Text: text}},
+		}
 	}
-	var segStart, segEnd int64
-	if len(words) > 0 {
-		segStart = words[0].Start
-		segEnd = words[len(words)-1].End
+
+	wantWords := wordsRequested(opts.TimestampGranularities)
+	segments := make([]*pb.TranscriptSegment, 0, len(groups))
+	for id, group := range groups {
+		parts := make([]string, len(group))
+		for i, gw := range group {
+			parts[i] = gw.W
+		}
+		seg := &pb.TranscriptSegment{
+			Id:     int32(id),
+			Start:  secondsToNanos(group[0].Start),
+			End:    secondsToNanos(group[len(group)-1].End),
+			Text:   strings.TrimSpace(strings.Join(parts, " ")),
+			Tokens: tokensInWindow(doc.Tokens, group[0].Start, group[len(group)-1].End),
+		}
+		if wantWords {
+			ws := make([]*pb.TranscriptWord, len(group))
+			for i, gw := range group {
+				ws[i] = &pb.TranscriptWord{Start: secondsToNanos(gw.Start), End: secondsToNanos(gw.End), Text: gw.W}
+			}
+			seg.Words = ws
+		}
+		segments = append(segments, seg)
 	}
-	seg := &pb.TranscriptSegment{Id: 0, Start: segStart, End: segEnd, Text: text, Tokens: tokens}
-	if wordsRequested(opts.TimestampGranularities) {
-		seg.Words = words
-	}
-	return pb.TranscriptResult{Text: text, Segments: []*pb.TranscriptSegment{seg}}
+	return pb.TranscriptResult{Text: text, Segments: segments}
 }
 
+// splitWordsIntoSegments groups words into segments exactly as NeMo's
+// get_segment_offsets does (nemo/collections/asr/parts/utils/timestamp_utils.py).
+// Walking the words, it closes a segment when (1) the gap rule is enabled
+// (gapSeconds > 0) and the segment already has words and the gap from the
+// previous word's end to this word's start is >= gapSeconds - the current word
+// then STARTS a new segment - or, checked only when the gap rule did not apply
+// (NeMo's elif), (2) the word ends with (or is) a separator, which closes the
+// segment INCLUDING that word. Trailing words flush into a final segment.
+// gapSeconds <= 0 disables the gap rule, matching NeMo's default
+// segment_gap_threshold=None (punctuation-only segments).
+func splitWordsIntoSegments(words []transcriptWord, separators []rune, gapSeconds float64) [][]transcriptWord {
+	var segments [][]transcriptWord
+	var cur []transcriptWord
+	for i, word := range words {
+		gapActive := gapSeconds > 0 && len(cur) > 0
+		if gapActive && (word.Start-words[i-1].End) >= gapSeconds {
+			segments = append(segments, cur)
+			cur = []transcriptWord{word}
+			continue
+		}
+		if !gapActive && endsWithSeparator(word.W, separators) {
+			cur = append(cur, word)
+			segments = append(segments, cur)
+			cur = nil
+			continue
+		}
+		cur = append(cur, word)
+	}
+	if len(cur) > 0 {
+		segments = append(segments, cur)
+	}
+	return segments
+}
+
+// endsWithSeparator reports whether w's last rune is in separators (matching
+// NeMo's `word[-1] in delims or word in delims`).
+func endsWithSeparator(w string, separators []rune) bool {
+	r := []rune(strings.TrimSpace(w))
+	if len(r) == 0 {
+		return false
+	}
+	last := r[len(r)-1]
+	for _, s := range separators {
+		if last == s {
+			return true
+		}
+	}
+	return false
+}
+
+// tokensInWindow returns the ids of tokens whose timestamp t falls in
+// [start, end] (inclusive), assigning each token to the segment that spans its
+// time. The last segment's end is the last word end, so the final token is
+// included.
+func tokensInWindow(tokens []transcriptToken, start, end float64) []int32 {
+	var ids []int32
+	for _, t := range tokens {
+		if t.T >= start && t.T <= end {
+			ids = append(ids, t.ID)
+		}
+	}
+	return ids
+}
+
+// streamSegmenter accumulates streaming words into per-utterance segments. EOU
+// is the model's own utterance boundary; each closed segment takes its start/end
+// from its first/last accumulated word.
+type streamSegmenter struct {
+	segs   []*pb.TranscriptSegment
+	cur    []transcriptWord
+	nextID int32
+}
+
+func (s *streamSegmenter) add(doc streamFeedJSON) {
+	s.cur = append(s.cur, doc.Words...)
+	// Close the segment on either turn signal: <EOU> (end of utterance) or
+	// <EOB> (backchannel). ABI v4 reported both via "eou"; v5 split them, so we
+	// OR them here to keep the v4 segmentation boundaries.
+	if doc.Eou != 0 || doc.Eob != 0 {
+		s.flush()
+	}
+}
+
+func (s *streamSegmenter) flush() {
+	if len(s.cur) == 0 {
+		return
+	}
+	parts := make([]string, len(s.cur))
+	for i, w := range s.cur {
+		parts[i] = w.W
+	}
+	s.segs = append(s.segs, &pb.TranscriptSegment{
+		Id:    s.nextID,
+		Start: secondsToNanos(s.cur[0].Start),
+		End:   secondsToNanos(s.cur[len(s.cur)-1].End),
+		Text:  strings.TrimSpace(strings.Join(parts, " ")),
+	})
+	s.nextID++
+	s.cur = nil
+}
+
+func (s *streamSegmenter) segments() []*pb.TranscriptSegment { return s.segs }
+
 // wordsRequested reports whether the caller asked for word-level timestamps.
 // The OpenAI transcription API gates word timings behind
 // timestamp_granularities[] containing "word" and defaults to segment-level
@@ -361,7 +560,12 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra
 		return status.Error(codes.Canceled, "transcription cancelled")
 	}
 
-	stream := CppStreamBegin(p.ctxPtr)
+	var stream uintptr
+	if CppStreamBeginLang != nil {
+		stream = CppStreamBeginLang(p.ctxPtr, opts.GetLanguage())
+	} else {
+		stream = CppStreamBegin(p.ctxPtr)
+	}
 	if stream == 0 {
 		// Not a cache-aware streaming model: run a normal offline
 		// transcription and emit it as one delta + a closing final result.
@@ -390,6 +594,14 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra
 		return err
 	}
 
+	// ABI v4: when the streaming JSON entry points are present, drive them so the
+	// per-utterance segments carry per-word start/end timestamps. Falls through to
+	// the text-only loop below against an older libparakeet.so. Runs under the
+	// engineMu already held above.
+	if CppStreamFeedJSON != nil {
+		return p.streamJSON(ctx, stream, data, duration, results)
+	}
+
 	var (
 		full     strings.Builder
 		segText  strings.Builder
@@ -466,6 +678,72 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra
 	return nil
 }
 
+// streamJSON drives the streaming JSON entry points (present since ABI v4): each
+// feed/finalize returns a {text,eou,eob,frame_sec,words} document. The
+// newly-finalized text is emitted as a delta (unchanged streaming contract)
+// while words are accumulated into per-utterance segments (closed on <EOU> or
+// <EOB>) so the closing FinalResult carries timestamped segments. Runs under
+// engineMu (already held by the caller).
+func (p *ParakeetCpp) streamJSON(ctx context.Context, stream uintptr, data []float32,
+	duration float32, results chan *pb.TranscriptStreamResponse) error {
+	var (
+		full strings.Builder
+		seg  streamSegmenter
+	)
+	// consume frees the malloc'd char* (a 0 return is an error), parses the JSON,
+	// emits the delta, and routes words through the segmenter.
+	consume := func(ret uintptr) error {
+		if ret == 0 {
+			msg := CppLastError(p.ctxPtr)
+			if msg == "" {
+				msg = "unknown error"
+			}
+			return fmt.Errorf("parakeet-cpp: stream feed/finalize failed: %s", msg)
+		}
+		raw := goStringFromCPtr(ret)
+		CppFreeString(ret)
+		var doc streamFeedJSON
+		if err := json.Unmarshal([]byte(raw), &doc); err != nil {
+			return fmt.Errorf("parakeet-cpp: decode stream json: %w", err)
+		}
+		if doc.Text != "" {
+			full.WriteString(doc.Text)
+			results <- &pb.TranscriptStreamResponse{Delta: doc.Text}
+		}
+		seg.add(doc)
+		return nil
+	}
+
+	for off := 0; off < len(data); off += streamChunkSamples {
+		if err := ctx.Err(); err != nil {
+			return status.Error(codes.Canceled, "transcription cancelled")
+		}
+		end := min(off+streamChunkSamples, len(data))
+		chunk := data[off:end]
+		if err := consume(CppStreamFeedJSON(stream, chunk, int32(len(chunk)))); err != nil {
+			return err
+		}
+	}
+	if err := consume(CppStreamFinalizeJSON(stream)); err != nil {
+		return err
+	}
+	seg.flush() // close any trailing utterance that never saw an EOU
+
+	text := strings.TrimSpace(full.String())
+	segments := seg.segments()
+	if len(segments) == 0 && text != "" {
+		segments = append(segments, &pb.TranscriptSegment{Id: 0, Text: text})
+	}
+	results <- &pb.TranscriptStreamResponse{
+		FinalResult: &pb.TranscriptResult{
+			Text:     text,
+			Segments: segments,
+			Duration: duration,
+		},
+	}
+	return nil
+}
+
 // decodeWavMono16k converts any input audio to 16 kHz mono PCM and returns the
 // float samples plus the clip duration in seconds. Mirrors the whisper
 // backend: utils.AudioToWav (ffmpeg) normalises rate/channels, go-audio

backend/go/parakeet-cpp/goparakeetcpp_test.go

@@ -53,6 +53,10 @@ func ensureLibLoaded() {
 		purego.RegisterLibFunc(&CppStreamFeed, lib, "parakeet_capi_stream_feed")
 		purego.RegisterLibFunc(&CppStreamFinalize, lib, "parakeet_capi_stream_finalize")
 		purego.RegisterLibFunc(&CppStreamFree, lib, "parakeet_capi_stream_free")
+		if sym, err := purego.Dlsym(lib, "parakeet_capi_stream_feed_json"); err == nil && sym != 0 {
+			purego.RegisterLibFunc(&CppStreamFeedJSON, lib, "parakeet_capi_stream_feed_json")
+			purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json")
+		}
 		purego.RegisterLibFunc(&CppFreeString, lib, "parakeet_capi_free_string")
 		purego.RegisterLibFunc(&CppLastError, lib, "parakeet_capi_last_error")
 	})
@@ -107,13 +111,22 @@ var _ = Describe("ParakeetCpp", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(strings.TrimSpace(res.Text)).ToNot(BeEmpty(),
 				"expected non-empty transcript for %s", audioPath)
-			Expect(res.Segments).To(HaveLen(1),
-				"synthesises a single whole-clip segment")
-			Expect(res.Segments[0].Text).To(Equal(res.Text),
-				"single segment text must equal the top-level text")
-			// Default (no granularities) is segment-level: no per-word timings.
-			Expect(res.Segments[0].Words).To(BeEmpty(),
-				"word timings are opt-in via timestamp_granularities")
+			// NeMo-faithful segmentation: one or more punctuation-delimited
+			// segments, each with text and a monotonically-advancing time span.
+			Expect(res.Segments).ToNot(BeEmpty(), "expected at least one segment")
+			var prevEnd int64
+			for i, seg := range res.Segments {
+				Expect(strings.TrimSpace(seg.Text)).ToNot(BeEmpty(),
+					"segment %d must have text", i)
+				Expect(seg.End).To(BeNumerically(">=", seg.Start),
+					"segment %d end must not precede its start", i)
+				Expect(seg.Start).To(BeNumerically(">=", prevEnd),
+					"segments must be in time order")
+				prevEnd = seg.End
+				// Default (no granularities) is segment-level: no per-word timings.
+				Expect(seg.Words).To(BeEmpty(),
+					"word timings are opt-in via timestamp_granularities")
+			}
 		})
 
 		It("emits word-level timestamps when granularity=word", func() {
@@ -129,15 +142,28 @@ var _ = Describe("ParakeetCpp", func() {
 				TimestampGranularities: []string{"word"},
 			})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Segments).To(HaveLen(1))
-			seg := res.Segments[0]
-			Expect(seg.Words).ToNot(BeEmpty(),
-				"expected per-word timestamps with granularity=word")
-			// Monotonic, non-negative timings spanning the segment.
-			Expect(seg.Words[0].Start).To(BeNumerically(">=", int64(0)))
-			Expect(seg.End).To(BeNumerically(">=", seg.Start))
-			Expect(seg.Words[len(seg.Words)-1].End).To(Equal(seg.End),
-				"segment end tracks the last word")
+			Expect(res.Segments).ToNot(BeEmpty())
+			// With word granularity every segment carries its own words, and each
+			// segment's span tracks its first/last word; word starts advance
+			// monotonically across the whole transcript.
+			totalWords := 0
+			var prevStart int64 = -1
+			for i, seg := range res.Segments {
+				Expect(seg.Words).ToNot(BeEmpty(),
+					"segment %d must carry per-word timestamps with granularity=word", i)
+				Expect(seg.Start).To(Equal(seg.Words[0].Start),
+					"segment %d start tracks its first word", i)
+				Expect(seg.End).To(Equal(seg.Words[len(seg.Words)-1].End),
+					"segment %d end tracks its last word", i)
+				for _, w := range seg.Words {
+					Expect(w.End).To(BeNumerically(">=", w.Start))
+					Expect(w.Start).To(BeNumerically(">=", prevStart))
+					prevStart = w.Start
+					totalWords++
+				}
+			}
+			Expect(totalWords).To(BeNumerically(">", 0))
+			Expect(res.Segments[0].Words[0].Start).To(BeNumerically(">=", int64(0)))
 		})
 	})
 

backend/go/parakeet-cpp/main.go

@@ -65,6 +65,25 @@ func main() {
 		purego.RegisterLibFunc(&CppTranscribePcmBatchJSON, lib, "parakeet_capi_transcribe_pcm_batch_json")
 	}
 
+	// Per-request language variants (multilingual nemotron). Same probe pattern:
+	// present only in libparakeet.so built with multilingual support, so the
+	// backend still loads against an older library and falls back to the
+	// non-lang batched + streaming entry points (model default / "auto").
+	if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_pcm_batch_json_lang"); err == nil && sym != 0 {
+		purego.RegisterLibFunc(&CppTranscribePcmBatchJSONLang, lib, "parakeet_capi_transcribe_pcm_batch_json_lang")
+	}
+	if sym, err := purego.Dlsym(lib, "parakeet_capi_stream_begin_lang"); err == nil && sym != 0 {
+		purego.RegisterLibFunc(&CppStreamBeginLang, lib, "parakeet_capi_stream_begin_lang")
+	}
+
+	// Streaming JSON entry points (ABI v4): surface per-word timestamps on the
+	// streaming path. Same probe pattern; absent in older libparakeet.so, where
+	// the backend falls back to the text-only streaming feed.
+	if sym, err := purego.Dlsym(lib, "parakeet_capi_stream_feed_json"); err == nil && sym != 0 {
+		purego.RegisterLibFunc(&CppStreamFeedJSON, lib, "parakeet_capi_stream_feed_json")
+		purego.RegisterLibFunc(&CppStreamFinalizeJSON, lib, "parakeet_capi_stream_finalize_json")
+	}
+
 	fmt.Fprintf(os.Stderr, "[parakeet-cpp] ABI=%d\n", CppAbiVersion())
 
 	flag.Parse()

backend/go/parakeet-cpp/segments_test.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func tw(text string, start, end float64) transcriptWord {
+	return transcriptWord{W: text, Start: start, End: end}
+}
+
+var _ = Describe("splitWordsIntoSegments (NeMo get_segment_offsets parity)", func() {
+	seps := []rune{'.', '?', '!'}
+
+	It("splits on sentence-ending punctuation, including the delimiter word", func() {
+		words := []transcriptWord{tw("hello", 0, 0.4), tw("world.", 0.4, 0.8), tw("bye", 1.0, 1.3)}
+		segs := splitWordsIntoSegments(words, seps, 0)
+		Expect(segs).To(HaveLen(2))
+		Expect(segs[0]).To(HaveLen(2))
+		Expect(segs[0][1].W).To(Equal("world."))
+		Expect(segs[1]).To(HaveLen(1))
+		Expect(segs[1][0].W).To(Equal("bye"))
+	})
+
+	It("keeps a single segment with no terminal punctuation and gap off", func() {
+		words := []transcriptWord{tw("a", 0, 0.2), tw("b", 0.2, 0.4), tw("c", 5.0, 5.2)}
+		segs := splitWordsIntoSegments(words, seps, 0)
+		Expect(segs).To(HaveLen(1))
+	})
+
+	It("splits on the gap rule when enabled, the gapped word starting the next segment", func() {
+		words := []transcriptWord{tw("a", 0, 0.2), tw("b", 0.2, 0.4), tw("c", 5.0, 5.2)}
+		segs := splitWordsIntoSegments(words, seps, 1.0) // c is 4.6s after b
+		Expect(segs).To(HaveLen(2))
+		Expect(segs[0]).To(HaveLen(2)) // a b
+		Expect(segs[1]).To(HaveLen(1)) // c
+		Expect(segs[1][0].W).To(Equal("c"))
+	})
+
+	It("checks the gap rule before punctuation (NeMo elif order)", func() {
+		// "b." would terminate, but c is far after it -> gap closes [a b.] at b.
+		words := []transcriptWord{tw("a", 0, 0.2), tw("b.", 0.2, 0.4), tw("c", 9.0, 9.2)}
+		segs := splitWordsIntoSegments(words, seps, 1.0)
+		Expect(segs).To(HaveLen(2))
+		Expect(segs[0]).To(HaveLen(2))
+		Expect(segs[1][0].W).To(Equal("c"))
+	})
+
+	It("still splits on punctuation when the gap rule is enabled but does not fire", func() {
+		words := []transcriptWord{tw("hi.", 0, 0.4), tw("bye", 0.4, 0.8)}
+		segs := splitWordsIntoSegments(words, seps, 5.0) // gap never reached
+		Expect(segs).To(HaveLen(2))
+		Expect(segs[0][0].W).To(Equal("hi."))
+	})
+
+	It("returns nothing for empty input", func() {
+		Expect(splitWordsIntoSegments(nil, seps, 0)).To(BeEmpty())
+	})
+})
+
+var _ = Describe("transcriptResultFromDoc (multi-segment)", func() {
+	doc := transcriptJSON{
+		Text:     "hello world. bye now",
+		FrameSec: 0.08,
+		Words: []transcriptWord{
+			{W: "hello", Start: 0.0, End: 0.4},
+			{W: "world.", Start: 0.4, End: 0.8},
+			{W: "bye", Start: 1.0, End: 1.3},
+			{W: "now", Start: 1.3, End: 1.6},
+		},
+		Tokens: []transcriptToken{{ID: 1, T: 0.1}, {ID: 2, T: 0.5}, {ID: 3, T: 1.1}, {ID: 4, T: 1.4}},
+	}
+
+	It("emits one segment per punctuation-delimited group with start/end", func() {
+		res := transcriptResultFromDoc(doc, &pb.TranscriptRequest{}, 0)
+		Expect(res.Segments).To(HaveLen(2))
+		Expect(res.Segments[0].Text).To(Equal("hello world."))
+		Expect(res.Segments[0].Start).To(Equal(int64(0)))
+		Expect(res.Segments[0].End).To(Equal(secondsToNanos(0.8)))
+		Expect(res.Segments[1].Text).To(Equal("bye now"))
+		Expect(res.Segments[1].Start).To(Equal(secondsToNanos(1.0)))
+		Expect(res.Segments[1].Id).To(Equal(int32(1)))
+	})
+
+	It("assigns tokens to the segment whose time window contains them", func() {
+		res := transcriptResultFromDoc(doc, &pb.TranscriptRequest{}, 0)
+		Expect(res.Segments[0].Tokens).To(Equal([]int32{1, 2}))
+		Expect(res.Segments[1].Tokens).To(Equal([]int32{3, 4}))
+	})
+
+	It("attaches per-segment words only when word granularity requested", func() {
+		plain := transcriptResultFromDoc(doc, &pb.TranscriptRequest{}, 0)
+		Expect(plain.Segments[0].Words).To(BeEmpty())
+		withWords := transcriptResultFromDoc(doc, &pb.TranscriptRequest{TimestampGranularities: []string{"word"}}, 0)
+		Expect(withWords.Segments[0].Words).To(HaveLen(2))
+	})
+
+	It("falls back to a single text segment when there are no words", func() {
+		res := transcriptResultFromDoc(transcriptJSON{Text: "hi"}, &pb.TranscriptRequest{}, 0)
+		Expect(res.Segments).To(HaveLen(1))
+		Expect(res.Segments[0].Text).To(Equal("hi"))
+	})
+})
+
+var _ = Describe("streaming segment assembly", func() {
+	It("closes a segment with start/end from its words on EOU", func() {
+		acc := &streamSegmenter{}
+		acc.add(streamFeedJSON{Text: "hello world", Eou: 1, Words: []transcriptWord{
+			{W: "hello", Start: 0.0, End: 0.4}, {W: "world", Start: 0.4, End: 0.9},
+		}})
+		segs := acc.segments()
+		Expect(segs).To(HaveLen(1))
+		Expect(segs[0].Text).To(Equal("hello world"))
+		Expect(segs[0].Start).To(Equal(int64(0)))
+		Expect(segs[0].End).To(Equal(secondsToNanos(0.9)))
+	})
+
+	It("buffers words across feeds until EOU", func() {
+		acc := &streamSegmenter{}
+		acc.add(streamFeedJSON{Text: "hi", Eou: 0, Words: []transcriptWord{{W: "hi", Start: 0, End: 0.3}}})
+		Expect(acc.segments()).To(BeEmpty())
+		acc.add(streamFeedJSON{Text: "there", Eou: 1, Words: []transcriptWord{{W: "there", Start: 0.3, End: 0.7}}})
+		Expect(acc.segments()).To(HaveLen(1))
+		Expect(acc.segments()[0].Text).To(Equal("hi there"))
+	})
+
+	// ABI v5 split <EOB> (backchannel) out of the "eou" flag into its own "eob"
+	// field; a backchannel must still close the segment as it did in v4.
+	It("closes a segment on EOB (backchannel) too", func() {
+		acc := &streamSegmenter{}
+		acc.add(streamFeedJSON{Text: "uh huh", Eou: 0, Eob: 1, Words: []transcriptWord{
+			{W: "uh", Start: 0.0, End: 0.2}, {W: "huh", Start: 0.2, End: 0.5},
+		}})
+		segs := acc.segments()
+		Expect(segs).To(HaveLen(1))
+		Expect(segs[0].Text).To(Equal("uh huh"))
+		Expect(segs[0].End).To(Equal(secondsToNanos(0.5)))
+	})
+})

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=1f9ee88e09c258053fa59d5e05e23dfb10fa0b13
+STABLEDIFFUSION_GGML_VERSION?=19bdfe22d255d5b4dff39d449318b9bc5ea2317f
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/stablediffusion-ggml/cpp/gosd.cpp

@@ -386,6 +386,7 @@ int load_model(const char *model, char *model_path, char* options[], int threads
     const char *llm_vision_path = "";
     const char *diffusion_model_path = stableDiffusionModel;
     const char *high_noise_diffusion_model_path = "";
+    const char *uncond_diffusion_model_path = "";
     const char *taesd_path  = "";
     const char *control_net_path = "";
     const char *embedding_dir = "";
@@ -472,6 +473,7 @@ int load_model(const char *model, char *model_path, char* options[], int threads
         if (!strcmp(optname, "llm_vision_path")) llm_vision_path = strdup(optval);
         if (!strcmp(optname, "diffusion_model_path")) diffusion_model_path = strdup(optval);
         if (!strcmp(optname, "high_noise_diffusion_model_path")) high_noise_diffusion_model_path = strdup(optval);
+        if (!strcmp(optname, "uncond_diffusion_model_path")) uncond_diffusion_model_path = strdup(optval);
         if (!strcmp(optname, "taesd_path")) taesd_path = strdup(optval);
         if (!strcmp(optname, "control_net_path")) control_net_path = strdup(optval);
         if (!strcmp(optname, "embedding_dir")) {
@@ -571,6 +573,7 @@ int load_model(const char *model, char *model_path, char* options[], int threads
     ctx_params.llm_vision_path = llm_vision_path;
     ctx_params.diffusion_model_path = diffusion_model_path;
     ctx_params.high_noise_diffusion_model_path = high_noise_diffusion_model_path;
+    ctx_params.uncond_diffusion_model_path = uncond_diffusion_model_path;
     ctx_params.vae_path = vae_path;
     ctx_params.audio_vae_path = audio_vae_path;
     ctx_params.embeddings_connectors_path = embeddings_connectors_path;

backend/go/vibevoice-cpp/CMakeLists.txt

@@ -26,8 +26,16 @@ add_library(govibevoicecpp MODULE cpp/govibevoicecpp.cpp)
 # vv_capi_* symbols (purego dlopens them by name, nothing in our
 # translation unit references them). Force the static archive's
 # entire contents into the MODULE so dlsym finds vv_capi_load etc.
+#
+# Link the `vibevoice` TARGET (not a bare archive path) so CMake builds
+# libvibevoice.a first and tracks the dependency: the upstream project is added
+# with EXCLUDE_FROM_ALL, so without a target-level link there is no rule to
+# build it. Passing only $<TARGET_FILE:vibevoice> as a path on Apple left the
+# build with "No rule to make target 'vibevoice/libvibevoice.a'" (issue #10267).
+# force_load is then applied as a separate link option.
 if(APPLE)
-    target_link_libraries(govibevoicecpp PRIVATE -Wl,-force_load $<TARGET_FILE:vibevoice>)
+    target_link_libraries(govibevoicecpp PRIVATE vibevoice)
+    target_link_options(govibevoicecpp PRIVATE "-Wl,-force_load,$<TARGET_FILE:vibevoice>")
 elseif(MSVC)
     target_link_libraries(govibevoicecpp PRIVATE vibevoice)
     set_property(TARGET govibevoicecpp APPEND PROPERTY LINK_FLAGS "/WHOLEARCHIVE:vibevoice")

backend/go/vibevoice-cpp/Makefile

@@ -94,26 +94,30 @@ purge:
 # Build all variants (Linux only)
 ifeq ($(UNAME_S),Linux)
 libgovibevoicecpp-avx.so: sources/vibevoice.cpp
+	$(MAKE) purge
 	$(info ${GREEN}I vibevoice-cpp build info:avx${RESET})
 	SO_TARGET=libgovibevoicecpp-avx.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgovibevoicecpp-custom
-	rm -rf build-libgovibevoicecpp-avx.so
+	rm -rfv build*
 
 libgovibevoicecpp-avx2.so: sources/vibevoice.cpp
+	$(MAKE) purge
 	$(info ${GREEN}I vibevoice-cpp build info:avx2${RESET})
 	SO_TARGET=libgovibevoicecpp-avx2.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgovibevoicecpp-custom
-	rm -rf build-libgovibevoicecpp-avx2.so
+	rm -rfv build*
 
 libgovibevoicecpp-avx512.so: sources/vibevoice.cpp
+	$(MAKE) purge
 	$(info ${GREEN}I vibevoice-cpp build info:avx512${RESET})
 	SO_TARGET=libgovibevoicecpp-avx512.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgovibevoicecpp-custom
-	rm -rf build-libgovibevoicecpp-avx512.so
+	rm -rfv build*
 endif
 
 # Build fallback variant (all platforms)
 libgovibevoicecpp-fallback.so: sources/vibevoice.cpp
+	$(MAKE) purge
 	$(info ${GREEN}I vibevoice-cpp build info:fallback${RESET})
 	SO_TARGET=libgovibevoicecpp-fallback.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgovibevoicecpp-custom
-	rm -rf build-libgovibevoicecpp-fallback.so
+	rm -rfv build*
 
 libgovibevoicecpp-custom: CMakeLists.txt cpp/govibevoicecpp.cpp cpp/govibevoicecpp.h
 	mkdir -p build-$(SO_TARGET) && \

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=99613cb720b65036237d44b52f753b51f75c2797
+WHISPER_CPP_VERSION?=df7638d8229a243af8a4b5a8ae557e0d74e0a0ae
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/index.yaml

@@ -337,6 +337,35 @@
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-rfdetr-cpp"
     intel: "intel-sycl-f32-rfdetr-cpp"
     vulkan: "vulkan-rfdetr-cpp"
+- &locateanything
+  name: "locate-anything"
+  alias: "locate-anything"
+  license: apache-2.0
+  description: |
+    Open-vocabulary object detection and visual grounding (NVIDIA
+    LocateAnything-3B) in C/C++ using GGML. Loads pre-built GGUF weights
+    and, given an image and a free-form text prompt, returns bounding
+    boxes, class labels, and confidence scores for the referred objects.
+  urls:
+    - https://github.com/mudler/locate-anything.cpp
+    - https://huggingface.co/nvidia/LocateAnything-3B
+  tags:
+    - object-detection
+    - visual-grounding
+    - open-vocabulary
+    - locate-anything
+    - gpu
+    - cpu
+  capabilities:
+    default: "cpu-locate-anything-cpp"
+    nvidia: "cuda12-locate-anything-cpp"
+    nvidia-cuda-12: "cuda12-locate-anything-cpp"
+    nvidia-cuda-13: "cuda13-locate-anything-cpp"
+    nvidia-l4t: "nvidia-l4t-arm64-locate-anything-cpp"
+    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-locate-anything-cpp"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-locate-anything-cpp"
+    intel: "intel-sycl-f32-locate-anything-cpp"
+    vulkan: "vulkan-locate-anything-cpp"
 - &vllm
   name: "vllm"
   license: apache-2.0
@@ -1225,6 +1254,7 @@
     default: "cpu-sherpa-onnx"
     nvidia: "cuda12-sherpa-onnx"
     nvidia-cuda-12: "cuda12-sherpa-onnx"
+    metal: "metal-sherpa-onnx"
 - !!merge <<: *neutts
   name: "neutts-development"
   capabilities:
@@ -1557,6 +1587,7 @@
     - localai/localai-backends:master-metal-darwin-arm64-kitten-tts
 - !!merge <<: *local-store
   name: "local-store-development"
+  alias: "local-store"
   uri: "quay.io/go-skynet/local-ai-backends:master-cpu-local-store"
   mirrors:
     - localai/localai-backends:master-cpu-local-store
@@ -1567,6 +1598,7 @@
     - localai/localai-backends:latest-metal-darwin-arm64-local-store
 - !!merge <<: *local-store
   name: "metal-local-store-development"
+  alias: "local-store"
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-local-store"
   mirrors:
     - localai/localai-backends:master-metal-darwin-arm64-local-store
@@ -4685,12 +4717,14 @@
     default: "cpu-speaker-recognition"
     nvidia: "cuda12-speaker-recognition"
     nvidia-cuda-12: "cuda12-speaker-recognition"
+    metal: "metal-speaker-recognition"
 - !!merge <<: *speakerrecognition
   name: "speaker-recognition-development"
   capabilities:
     default: "cpu-speaker-recognition-development"
     nvidia: "cuda12-speaker-recognition-development"
     nvidia-cuda-12: "cuda12-speaker-recognition-development"
+    metal: "metal-speaker-recognition-development"
 - !!merge <<: *speakerrecognition
   name: "cpu-speaker-recognition"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-speaker-recognition"
@@ -4711,6 +4745,16 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-speaker-recognition"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-12-speaker-recognition
+- !!merge <<: *speakerrecognition
+  name: "metal-speaker-recognition"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-speaker-recognition"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-speaker-recognition
+- !!merge <<: *speakerrecognition
+  name: "metal-speaker-recognition-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-speaker-recognition"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-speaker-recognition
 ## sherpa-onnx
 - !!merge <<: *sherpa-onnx
   name: "sherpa-onnx-development"
@@ -4718,6 +4762,7 @@
     default: "cpu-sherpa-onnx-development"
     nvidia: "cuda12-sherpa-onnx-development"
     nvidia-cuda-12: "cuda12-sherpa-onnx-development"
+    metal: "metal-sherpa-onnx-development"
 - !!merge <<: *sherpa-onnx
   name: "cpu-sherpa-onnx"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-sherpa-onnx"
@@ -4738,3 +4783,13 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-sherpa-onnx"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-12-sherpa-onnx
+- !!merge <<: *sherpa-onnx
+  name: "metal-sherpa-onnx"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-sherpa-onnx"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-sherpa-onnx
+- !!merge <<: *sherpa-onnx
+  name: "metal-sherpa-onnx-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-sherpa-onnx"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-sherpa-onnx

backend/python/common/python_utils.py

@@ -5,6 +5,31 @@ imported by any backend that needs to parse LocalAI gRPC options or build a
 chat-template-compatible message list from proto Message objects.
 """
 import json
+from urllib.parse import unquote
+
+
+def resolve_model_path(model, model_file=""):
+    """Resolve a LocalAI model reference to something an HF/MLX loader accepts.
+
+    LocalAI hands backends either a plain HuggingFace repo id
+    (``namespace/name``), an already-local filesystem path, or a
+    ``file://`` URI (its ``LocalPrefix``) for models imported from disk.
+    Loaders such as ``mlx_lm.load`` reject the ``file://`` form because the
+    scheme is neither a valid repo id nor an existing path, so we normalize
+    it here before loading.
+
+    Resolution order:
+      1. Prefer ``model_file`` when set and non-empty - that is the resolved
+         local path LocalAI computed for the model.
+      2. Strip a ``file://`` scheme and percent-decode it to a plain path.
+      3. Leave plain repo ids and already-local paths unchanged.
+    """
+    candidate = model_file if model_file else model
+    if candidate is None:
+        return candidate
+    if candidate.startswith("file://"):
+        return unquote(candidate[len("file://"):])
+    return candidate
 
 
 def parse_options(options_list):

backend/python/mlx-distributed/backend.py

@@ -28,7 +28,7 @@ import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
-from python_utils import messages_to_dicts, parse_options as _shared_parse_options
+from python_utils import messages_to_dicts, parse_options as _shared_parse_options, resolve_model_path
 from mlx_utils import parse_tool_calls, split_reasoning
 
 
@@ -99,7 +99,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             from mlx_lm import load
             from mlx_lm.models.cache import make_prompt_cache, can_trim_prompt_cache, trim_prompt_cache
 
-            print(f"[Rank 0] Loading model: {request.Model}", file=sys.stderr)
+            # Normalize the model reference: strip LocalAI's file:// LocalPrefix
+            # and prefer the resolved ModelFile so mlx_lm.load() gets a plain
+            # repo id or filesystem path (it rejects file:// URIs).
+            model_path = resolve_model_path(request.Model, request.ModelFile)
+            print(f"[Rank 0] Loading model: {model_path}", file=sys.stderr)
 
             self.options = parse_options(request.Options)
             print(f"Options: {self.options}", file=sys.stderr)
@@ -128,7 +132,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 )
                 self.coordinator = DistributedCoordinator(self.group)
                 self.coordinator.broadcast_command(CMD_LOAD_MODEL)
-                self.coordinator.broadcast_model_name(request.Model)
+                self.coordinator.broadcast_model_name(model_path)
             else:
                 print("[Rank 0] No hostfile configured, running single-node", file=sys.stderr)
 
@@ -144,9 +148,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
             if tokenizer_config:
                 print(f"Loading with tokenizer_config: {tokenizer_config}", file=sys.stderr)
-                self.model, self.tokenizer = load(request.Model, tokenizer_config=tokenizer_config)
+                self.model, self.tokenizer = load(model_path, tokenizer_config=tokenizer_config)
             else:
-                self.model, self.tokenizer = load(request.Model)
+                self.model, self.tokenizer = load(model_path)
 
             if self.group is not None:
                 from sharding import pipeline_auto_parallel
@@ -157,7 +161,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 from mlx_cache import ThreadSafeLRUPromptCache
                 max_cache_entries = self.options.get("max_cache_entries", 10)
                 self.max_kv_size = self.options.get("max_kv_size", None)
-                self.model_key = request.Model
+                self.model_key = model_path
                 self.lru_cache = ThreadSafeLRUPromptCache(
                     max_size=max_cache_entries,
                     can_trim_fn=can_trim_prompt_cache,

backend/python/mlx-vlm/backend.py

@@ -18,7 +18,7 @@ import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
-from python_utils import messages_to_dicts, parse_options
+from python_utils import messages_to_dicts, parse_options, resolve_model_path
 from mlx_utils import parse_tool_calls, split_reasoning
 
 from mlx_vlm import load, stream_generate
@@ -67,7 +67,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             backend_pb2.Result: The load model result.
         """
         try:
-            print(f"Loading MLX-VLM model: {request.Model}", file=sys.stderr)
+            # Normalize the model reference: strip LocalAI's file:// LocalPrefix
+            # and prefer the resolved ModelFile so mlx_vlm.load() gets a plain
+            # repo id or filesystem path (it rejects file:// URIs).
+            model_path = resolve_model_path(request.Model, request.ModelFile)
+            print(f"Loading MLX-VLM model: {model_path}", file=sys.stderr)
             print(f"Request: {request}", file=sys.stderr)
 
             # Parse Options[] key:value strings into a typed dict
@@ -76,10 +80,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
             # Load model and processor using MLX-VLM
             # mlx-vlm load function returns (model, processor) instead of (model, tokenizer)
-            self.model, self.processor = load(request.Model)
+            self.model, self.processor = load(model_path)
 
             # Load model config for chat template support
-            self.config = load_config(request.Model)
+            self.config = load_config(model_path)
 
             # Auto-infer the tool parser from the chat template. mlx-vlm has
             # its own _infer_tool_parser that falls back to mlx-lm parsers.

backend/python/mlx/backend.py

@@ -17,7 +17,7 @@ import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
-from python_utils import messages_to_dicts, parse_options
+from python_utils import messages_to_dicts, parse_options, resolve_model_path
 from mlx_utils import parse_tool_calls, split_reasoning
 
 from mlx_lm import load, stream_generate
@@ -63,7 +63,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             backend_pb2.Result: The load model result.
         """
         try:
-            print(f"Loading MLX model: {request.Model}", file=sys.stderr)
+            # Normalize the model reference: strip LocalAI's file:// LocalPrefix
+            # and prefer the resolved ModelFile so mlx_lm.load() gets a plain
+            # repo id or filesystem path (it rejects file:// URIs).
+            model_path = resolve_model_path(request.Model, request.ModelFile)
+            print(f"Loading MLX model: {model_path}", file=sys.stderr)
             print(f"Request: {request}", file=sys.stderr)
 
             # Parse Options[] key:value strings into a typed dict (shared helper)
@@ -89,9 +93,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # Load model and tokenizer using MLX
             if tokenizer_config:
                 print(f"Loading with tokenizer_config: {tokenizer_config}", file=sys.stderr)
-                self.model, self.tokenizer = load(request.Model, tokenizer_config=tokenizer_config)
+                self.model, self.tokenizer = load(model_path, tokenizer_config=tokenizer_config)
             else:
-                self.model, self.tokenizer = load(request.Model)
+                self.model, self.tokenizer = load(model_path)
 
             # mlx_lm.load() returns a TokenizerWrapper that detects tool
             # calling and thinking markers from the chat template / vocab.
@@ -111,7 +115,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # Initialize thread-safe LRU prompt cache for efficient generation
             max_cache_entries = self.options.get("max_cache_entries", 10)
             self.max_kv_size = self.options.get("max_kv_size", None)
-            self.model_key = request.Model
+            self.model_key = model_path
             self.lru_cache = ThreadSafeLRUPromptCache(
                 max_size=max_cache_entries,
                 can_trim_fn=can_trim_prompt_cache,
@@ -407,6 +411,24 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         if not request.Prompt and request.UseTokenizerTemplate and request.Messages:
             messages = messages_to_dicts(request.Messages)
 
+            # The mlx-lm tokenizer only carries a text-LM chat template. A
+            # vision-language checkpoint (e.g. gemma-4 E4B) loaded here has no
+            # usable template, so apply_chat_template silently passes the raw
+            # text through and the model just echoes/loops (issue #10269).
+            # Warn loudly so the misroute is visible; such models belong on the
+            # mlx-vlm backend.
+            chat_template = getattr(self.tokenizer, "chat_template", None)
+            if not chat_template:
+                underlying = getattr(self.tokenizer, "_tokenizer", None)
+                chat_template = getattr(underlying, "chat_template", None)
+            if not chat_template:
+                print(
+                    "WARNING: this model has no chat template; output may be "
+                    "degenerate. Vision-language models (e.g. gemma-4 E4B) must "
+                    "use the 'mlx-vlm' backend instead of 'mlx'.",
+                    file=sys.stderr,
+                )
+
             kwargs = {"tokenize": False, "add_generation_prompt": True}
             if request.Tools:
                 try:

backend/python/mlx/test.py

@@ -12,7 +12,7 @@ import backend_pb2_grpc
 # Make the shared helpers importable so we can unit-test them without a
 # running gRPC server.
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
-from python_utils import messages_to_dicts, parse_options
+from python_utils import messages_to_dicts, parse_options, resolve_model_path
 from mlx_utils import parse_tool_calls, split_reasoning
 
 class TestBackendServicer(unittest.TestCase):
@@ -322,6 +322,42 @@ class TestSharedHelpers(unittest.TestCase):
         self.assertEqual(r, "")
         self.assertEqual(c, "just text")
 
+    def test_resolve_model_path_file_uri(self):
+        # file:// LocalPrefix (LocalAI import) is stripped to a plain path.
+        self.assertEqual(resolve_model_path("file:///a/b"), "/a/b")
+
+    def test_resolve_model_path_file_uri_percent_decoded(self):
+        # Percent-encoded characters (e.g. spaces) are decoded.
+        self.assertEqual(
+            resolve_model_path("file:///Users/me/My%20Models/Qwen3"),
+            "/Users/me/My Models/Qwen3",
+        )
+
+    def test_resolve_model_path_hf_repo_id_unchanged(self):
+        # Plain HuggingFace repo ids must pass through untouched.
+        self.assertEqual(
+            resolve_model_path("mlx-community/Qwen3-Coder-30B"),
+            "mlx-community/Qwen3-Coder-30B",
+        )
+
+    def test_resolve_model_path_local_path_unchanged(self):
+        # An already-local absolute path is left as-is.
+        self.assertEqual(resolve_model_path("/models/Qwen3"), "/models/Qwen3")
+
+    def test_resolve_model_path_prefers_model_file(self):
+        # The resolved ModelFile wins over Model when both are set.
+        self.assertEqual(
+            resolve_model_path("file:///ignored", "/resolved/local/path"),
+            "/resolved/local/path",
+        )
+
+    def test_resolve_model_path_model_file_file_uri(self):
+        # A ModelFile that is itself a file:// URI is also normalized.
+        self.assertEqual(
+            resolve_model_path("ignored", "file:///a/b"),
+            "/a/b",
+        )
+
     def test_parse_tool_calls_with_shim(self):
         tm = types.SimpleNamespace(
             tool_call_start="<tool_call>",

backend/python/speaker-recognition/requirements-mps.txt

@@ -0,0 +1,5 @@
+torch
+torchaudio
+speechbrain
+transformers
+onnxruntime

backend/python/vllm/backend.py

@@ -26,7 +26,10 @@ from vllm.engine.arg_utils import AsyncEngineArgs
 from vllm.engine.async_llm_engine import AsyncLLMEngine
 from vllm.sampling_params import SamplingParams
 from vllm.utils import random_uuid
-from vllm.transformers_utils.tokenizer import get_tokenizer
+try:
+    from vllm.tokenizers import get_tokenizer  # vLLM >= 0.22
+except ImportError:
+    from vllm.transformers_utils.tokenizer import get_tokenizer  # vLLM < 0.22
 from vllm.multimodal.utils import fetch_image
 from vllm.assets.video import VideoAsset
 import base64
@@ -147,9 +150,24 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 d["reasoning_content"] = msg.reasoning_content
             if msg.tool_calls:
                 try:
-                    d["tool_calls"] = json.loads(msg.tool_calls)
+                    tool_calls = json.loads(msg.tool_calls)
                 except json.JSONDecodeError:
                     pass
+                else:
+                    # OpenAI wire format carries function.arguments as a
+                    # JSON-encoded string, but chat templates (e.g. Qwen3)
+                    # iterate over it as a mapping. vLLM's own OpenAI server
+                    # parses arguments before applying the template, so do
+                    # the same here.
+                    if isinstance(tool_calls, list):
+                        for tc in tool_calls:
+                            func = tc.get("function") if isinstance(tc, dict) else None
+                            if isinstance(func, dict) and isinstance(func.get("arguments"), str):
+                                try:
+                                    func["arguments"] = json.loads(func["arguments"])
+                                except json.JSONDecodeError:
+                                    pass
+                    d["tool_calls"] = tool_calls
             result.append(d)
         return result
 

core/application/mitm.go

@@ -11,6 +11,29 @@ import (
 	"github.com/mudler/xlog"
 )
 
+// startMITMIfConfigured brings up the cloudproxy MITM listener when an
+// address is configured, treating any startup failure as non-fatal.
+//
+// The listener is opt-in middleware whose address is persisted in runtime
+// settings (/api/settings → runtime_settings.json) and replayed on every
+// boot. A bad value — e.g. a host the process can't bind, like a LAN IP
+// inside a container — must NOT abort the whole server: doing so crash-loops
+// with no way out, because the Settings UI used to correct the address can't
+// load if startup never completes. So on failure we log loudly and carry on;
+// the admin fixes the address via /api/settings, which calls RestartMITM.
+func startMITMIfConfigured(app *Application, options *config.ApplicationConfig) {
+	if options.MITMListen == "" {
+		return
+	}
+	if err := startMITMProxy(app, options); err != nil {
+		xlog.Error("mitm: cloudproxy listener failed to start — continuing without it",
+			"listen", options.MITMListen,
+			"error", err,
+			"hint", "fix the address via Settings (e.g. \":8082\" to bind all interfaces) and the listener will restart",
+		)
+	}
+}
+
 func startMITMProxy(app *Application, options *config.ApplicationConfig) error {
 	app.mitmMutex.Lock()
 	defer app.mitmMutex.Unlock()

core/application/mitm_test.go

@@ -0,0 +1,58 @@
+package application
+
+import (
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/system"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// minimal Application wired enough for startMITMProxy: an empty model
+// config loader (no host claims), CA written under a temp DataPath.
+func newMITMTestApp(dataPath string) (*Application, *config.ApplicationConfig) {
+	state, err := system.GetSystemState()
+	Expect(err).NotTo(HaveOccurred())
+	state.Model.ModelsPath = dataPath
+	opts := config.NewApplicationConfig(
+		config.WithSystemState(state),
+		config.WithDataPath(dataPath),
+	)
+	return newApplication(opts), opts
+}
+
+var _ = Describe("startMITMIfConfigured", func() {
+	It("does nothing when no listen address is configured", func() {
+		app, opts := newMITMTestApp(GinkgoT().TempDir())
+		opts.MITMListen = ""
+
+		Expect(func() { startMITMIfConfigured(app, opts) }).NotTo(Panic())
+		Expect(app.mitmServer.Load()).To(BeNil(), "no listener should be stored when disabled")
+	})
+
+	// Regression: a persisted-but-unbindable MITM address (e.g. a LAN host
+	// inside a container) must not abort startup. startMITMIfConfigured
+	// swallows the bind error so the rest of LocalAI still comes up and the
+	// admin can fix the address via the Settings UI.
+	It("logs and continues when the listen address cannot be bound", func() {
+		app, opts := newMITMTestApp(GinkgoT().TempDir())
+		// 192.0.2.1 is TEST-NET-1 (RFC 5737): guaranteed not assigned to any
+		// local interface, so bind fails deterministically without DNS.
+		opts.MITMListen = "192.0.2.1:8082"
+
+		Expect(func() { startMITMIfConfigured(app, opts) }).NotTo(Panic())
+		Expect(app.mitmServer.Load()).To(BeNil(), "failed listener must not be stored")
+	})
+
+	It("starts and stores the listener on a bindable address", func() {
+		app, opts := newMITMTestApp(GinkgoT().TempDir())
+		opts.MITMListen = "127.0.0.1:0" // OS-assigned free port
+
+		startMITMIfConfigured(app, opts)
+
+		srv := app.mitmServer.Load()
+		Expect(srv).NotTo(BeNil(), "listener should be stored on success")
+		DeferCleanup(srv.Stop)
+		Expect(srv.Addr()).NotTo(BeEmpty())
+	})
+})

core/application/router_factories.go

@@ -1,63 +1,120 @@
 package application
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/mudler/LocalAI/core/backend"
 	"github.com/mudler/LocalAI/core/config"
 )
 
-// adapterConfig resolves a model name to its runtime ModelConfig, or
-// nil when the name is unknown. Shared by the router-facing factories
-// below and by ModelConfigLookup.
+// adapterConfig resolves a model name to its runtime ModelConfig, or nil when
+// unknown. LoadModelConfigFileByNameDefaultOptions never returns nil — for an
+// unknown name it returns a defaults-filled stub with an empty Name (the YAML
+// `name:` field is required by Validate), which is how we tell the two apart.
 func (a *Application) adapterConfig(modelName string) *config.ModelConfig {
 	cfg, err := a.backendLoader.LoadModelConfigFileByNameDefaultOptions(modelName, a.applicationConfig)
-	if err != nil || cfg == nil {
+	if err != nil || cfg == nil || cfg.Name == "" {
 		return nil
 	}
 	return cfg
 }
 
-// ModelConfigLookup is the lookup function the router middleware's
-// classifier validator uses to confirm classifier_model declares
-// FLAG_SCORE before binding it.
+// ModelConfigLookup is the lookup the router middleware's classifier validator
+// uses to confirm classifier_model declares FLAG_SCORE before binding it.
 func (a *Application) ModelConfigLookup() func(modelName string) *config.ModelConfig {
 	return a.adapterConfig
 }
 
-// Scorer returns a backend.Scorer bound to the named model, or nil
-// when the model is unknown. Used as a method value (app.Scorer) by
-// router.ClassifierDeps — no factory-of-factory wrapper needed.
+// The router-facing factories below (Scorer, Embedder, Reranker, TokenCounter)
+// bind a model NAME at construction and re-resolve the CONFIG on every call.
+// Capturing the config at construction would bake in whatever state
+// adapterConfig saw first — including a stub returned before the YAML reached
+// bcl.configs (e.g. /import-model or gallery install racing startup). The
+// classifier registry caches factories by router-config fingerprint, so a
+// once-stale capture stays stale until the router config is edited.
+
 func (a *Application) Scorer(modelName string) backend.Scorer {
-	cfg := a.adapterConfig(modelName)
-	if cfg == nil {
+	if a.adapterConfig(modelName) == nil {
 		return nil
 	}
-	return backend.NewScorer(a.modelLoader, *cfg, a.applicationConfig)
+	return &lazyScorer{app: a, modelName: modelName}
+}
+
+type lazyScorer struct {
+	app       *Application
+	modelName string
+}
+
+func (l *lazyScorer) Score(ctx context.Context, prompt string, candidates []string) ([]backend.CandidateScore, error) {
+	cfg := l.app.adapterConfig(l.modelName)
+	if cfg == nil {
+		return nil, fmt.Errorf("scorer: model %q no longer available", l.modelName)
+	}
+	return backend.NewScorer(l.app.modelLoader, *cfg, l.app.applicationConfig).Score(ctx, prompt, candidates)
+}
+
+// TokenCounter returns a func so the middleware's literal field type accepts
+// it as a method value without importing core/http/middleware from here.
+func (a *Application) TokenCounter(modelName string) func(string) (int, error) {
+	if a.adapterConfig(modelName) == nil {
+		return nil
+	}
+	return func(text string) (int, error) {
+		cfg := a.adapterConfig(modelName)
+		if cfg == nil {
+			return 0, fmt.Errorf("token counter: model %q no longer available", modelName)
+		}
+		resp, err := backend.ModelTokenize(text, a.modelLoader, *cfg, a.applicationConfig)
+		if err != nil {
+			return 0, err
+		}
+		return len(resp.Tokens), nil
+	}
 }
 
-// Reranker returns a backend.Reranker bound to the named model, or
-// nil when unknown. The reranker model's `type:` (e.g. "colbert")
-// selects the scoring head inside the rerankers backend.
 func (a *Application) Reranker(modelName string) backend.Reranker {
-	cfg := a.adapterConfig(modelName)
-	if cfg == nil {
+	if a.adapterConfig(modelName) == nil {
 		return nil
 	}
-	return backend.NewReranker(a.modelLoader, *cfg, a.applicationConfig)
+	return &lazyReranker{app: a, modelName: modelName}
+}
+
+type lazyReranker struct {
+	app       *Application
+	modelName string
+}
+
+func (l *lazyReranker) Rerank(ctx context.Context, query string, documents []string) ([]backend.RerankResult, error) {
+	cfg := l.app.adapterConfig(l.modelName)
+	if cfg == nil {
+		return nil, fmt.Errorf("reranker: model %q no longer available", l.modelName)
+	}
+	return backend.NewReranker(l.app.modelLoader, *cfg, l.app.applicationConfig).Rerank(ctx, query, documents)
 }
 
-// Embedder returns a backend.Embedder bound to the named model, or
-// nil when unknown. Used by the router's L2 embedding cache.
 func (a *Application) Embedder(modelName string) backend.Embedder {
-	cfg := a.adapterConfig(modelName)
-	if cfg == nil {
+	if a.adapterConfig(modelName) == nil {
 		return nil
 	}
-	return backend.NewEmbedder(a.modelLoader, *cfg, a.applicationConfig)
+	return &lazyEmbedder{app: a, modelName: modelName}
 }
 
-// VectorStore returns a backend.VectorStore for the named collection,
-// or nil when the name is empty. Each router model gets its own
-// backend process via the model loader's cache keyed by storeName.
+type lazyEmbedder struct {
+	app       *Application
+	modelName string
+}
+
+func (l *lazyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
+	cfg := l.app.adapterConfig(l.modelName)
+	if cfg == nil {
+		return nil, fmt.Errorf("embedder: model %q no longer available", l.modelName)
+	}
+	return backend.NewEmbedder(l.app.modelLoader, *cfg, l.app.applicationConfig).Embed(ctx, text)
+}
+
+// VectorStore takes a store name, not a model name — no adapterConfig, no
+// staleness to avoid.
 func (a *Application) VectorStore(storeName string) backend.VectorStore {
 	return backend.NewVectorStore(a.modelLoader, a.applicationConfig, storeName)
 }

core/application/router_factories_test.go

@@ -0,0 +1,155 @@
+package application
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Regression: the router-facing factories used to capture
+// *config.ModelConfig at construction. A gallery install that raced
+// startup left a stub (Backend="") bound for the lifetime of the
+// classifier registry's cache entry, bypassing the user's `backend:`
+// config. These specs pin the lazy re-resolve.
+var _ = Describe("router_factories lazy config resolution", func() {
+	var (
+		tmpDir string
+		app    *Application
+	)
+
+	BeforeEach(func() {
+		var err error
+		tmpDir, err = os.MkdirTemp("", "router-factories-*")
+		Expect(err).NotTo(HaveOccurred())
+
+		appCfg := &config.ApplicationConfig{
+			Context:     context.Background(),
+			SystemState: &system.SystemState{Model: system.Model{ModelsPath: tmpDir}},
+		}
+		app = &Application{
+			backendLoader:     config.NewModelConfigLoader(tmpDir),
+			modelLoader:       model.NewModelLoader(appCfg.SystemState),
+			applicationConfig: appCfg,
+		}
+	})
+
+	AfterEach(func() {
+		_ = os.RemoveAll(tmpDir)
+	})
+
+	// writeCfg seeds both the on-disk YAML and the in-memory cache —
+	// removing only the cache would fall through to file-read.
+	writeCfg := func(name, backend string) {
+		yaml := "name: " + name + "\nbackend: " + backend + "\nparameters:\n  model: " + name + ".bin\n"
+		Expect(os.WriteFile(filepath.Join(tmpDir, name+".yaml"), []byte(yaml), 0644)).To(Succeed())
+		Expect(app.backendLoader.LoadModelConfigsFromPath(tmpDir)).To(Succeed())
+		cfg, ok := app.backendLoader.GetModelConfig(name)
+		Expect(ok).To(BeTrue(), "config must be loaded before the spec runs")
+		Expect(cfg.Backend).To(Equal(backend))
+	}
+
+	// removeCfg purges both the cache and the YAML so LoadModelConfigFileByName
+	// returns the empty-stub case and adapterConfig returns nil.
+	removeCfg := func(name string) {
+		app.backendLoader.RemoveModelConfig(name)
+		Expect(os.Remove(filepath.Join(tmpDir, name+".yaml"))).To(Succeed())
+	}
+
+	Context("Embedder", func() {
+		It("returns nil at construction for an unknown model", func() {
+			Expect(app.Embedder("missing")).To(BeNil())
+		})
+
+		It("re-resolves the model config on each Embed call", func() {
+			writeCfg("emb-test", "llama-cpp")
+			emb := app.Embedder("emb-test")
+			Expect(emb).NotTo(BeNil())
+
+			// The factory must hold the NAME, not a captured config —
+			// otherwise stale captures survive cache invalidation.
+			lazy, ok := emb.(*lazyEmbedder)
+			Expect(ok).To(BeTrue(), "Embedder must return *lazyEmbedder")
+			Expect(lazy.modelName).To(Equal("emb-test"))
+
+			// Mutate the cached config. A lazy implementation sees the
+			// update on the next adapterConfig call; a captured-at-
+			// construction implementation would still see "llama-cpp".
+			app.backendLoader.UpdateModelConfig("emb-test", func(c *config.ModelConfig) {
+				c.Backend = "rerankers"
+			})
+			Expect(lazy.app.adapterConfig("emb-test").Backend).To(Equal("rerankers"))
+
+			// Remove the config entirely → Embed must surface the disappearance.
+			removeCfg("emb-test")
+			_, err := emb.Embed(context.Background(), "anything")
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no longer available"))
+		})
+	})
+
+	Context("Scorer", func() {
+		It("returns nil at construction for an unknown model", func() {
+			Expect(app.Scorer("missing")).To(BeNil())
+		})
+
+		It("re-resolves the model config on each Score call", func() {
+			writeCfg("score-test", "llama-cpp")
+			sc := app.Scorer("score-test")
+			Expect(sc).NotTo(BeNil())
+
+			lazy, ok := sc.(*lazyScorer)
+			Expect(ok).To(BeTrue(), "Scorer must return *lazyScorer")
+			Expect(lazy.modelName).To(Equal("score-test"))
+
+			removeCfg("score-test")
+			_, err := sc.Score(context.Background(), "prompt", []string{"a"})
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no longer available"))
+		})
+	})
+
+	Context("Reranker", func() {
+		It("returns nil at construction for an unknown model", func() {
+			Expect(app.Reranker("missing")).To(BeNil())
+		})
+
+		It("re-resolves the model config on each Rerank call", func() {
+			writeCfg("rerank-test", "rerankers")
+			rr := app.Reranker("rerank-test")
+			Expect(rr).NotTo(BeNil())
+
+			lazy, ok := rr.(*lazyReranker)
+			Expect(ok).To(BeTrue(), "Reranker must return *lazyReranker")
+			Expect(lazy.modelName).To(Equal("rerank-test"))
+
+			removeCfg("rerank-test")
+			_, err := rr.Rerank(context.Background(), "q", []string{"d"})
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no longer available"))
+		})
+	})
+
+	Context("TokenCounter", func() {
+		It("returns nil at construction for an unknown model", func() {
+			Expect(app.TokenCounter("missing")).To(BeNil())
+		})
+
+		It("re-resolves the model config on each call", func() {
+			writeCfg("tok-test", "llama-cpp")
+			tc := app.TokenCounter("tok-test")
+			Expect(tc).NotTo(BeNil())
+
+			removeCfg("tok-test")
+			_, err := tc("anything")
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no longer available"))
+		})
+	})
+})

core/application/startup.go

@@ -23,9 +23,9 @@ import (
 	"github.com/mudler/LocalAI/core/services/routing/pii"
 	"github.com/mudler/LocalAI/core/services/routing/router"
 	"github.com/mudler/LocalAI/core/services/storage"
-	"github.com/mudler/LocalAI/pkg/signals"
 	coreStartup "github.com/mudler/LocalAI/core/startup"
 	"github.com/mudler/LocalAI/internal"
+	"github.com/mudler/LocalAI/pkg/signals"
 	"github.com/mudler/LocalAI/pkg/vram"
 
 	"github.com/mudler/LocalAI/pkg/model"
@@ -308,10 +308,31 @@ func New(opts ...config.AppOption) (*Application, error) {
 			application.galleryService.SetNATSClient(distSvc.Nats)
 			if distSvc.DistStores != nil && distSvc.DistStores.Gallery != nil {
 				// Clean up stale in-progress operations from previous crashed instances
-				if err := distSvc.DistStores.Gallery.CleanStale(30 * time.Minute); err != nil {
+				if _, err := distSvc.DistStores.Gallery.CleanStale(30 * time.Minute); err != nil {
 					xlog.Warn("Failed to clean stale gallery operations", "error", err)
 				}
 				application.galleryService.SetGalleryStore(distSvc.DistStores.Gallery)
+
+				// Reap stale ops periodically, not just at boot: an op orphaned by
+				// a replica that died mid-install (its foreground handler goroutine
+				// gone) would otherwise linger "processing" in the UI until the next
+				// restart. 30m matches the install/upgrade ceiling so a genuinely
+				// slow op is never reaped out from under itself.
+				gsvc := application.galleryService
+				go func() {
+					ticker := time.NewTicker(15 * time.Minute)
+					defer ticker.Stop()
+					for {
+						select {
+						case <-options.Context.Done():
+							return
+						case <-ticker.C:
+							if _, err := gsvc.ReapStaleOperations(30 * time.Minute); err != nil {
+								xlog.Warn("Failed to reap stale gallery operations", "error", err)
+							}
+						}
+					}
+				}()
 			}
 			// Hydrate from the store first so the wildcard subscriber finds an
 			// already-populated statuses map for any operations still in flight
@@ -441,11 +462,7 @@ func New(opts ...config.AppOption) (*Application, error) {
 	// traffic doesn't need a parallel config for MITM traffic.
 	// Runs after loadRuntimeSettingsFromFile so a listener configured
 	// via /api/settings is brought back up across restarts.
-	if options.MITMListen != "" {
-		if err := startMITMProxy(application, options); err != nil {
-			return nil, fmt.Errorf("mitm: startup: %w", err)
-		}
-	}
+	startMITMIfConfigured(application, options)
 
 	application.ModelLoader().SetBackendLoggingEnabled(options.EnableBackendLogging)
 

core/application/upgrade_checker.go

@@ -214,7 +214,9 @@ func (uc *UpgradeChecker) runCheck(ctx context.Context) {
 				"from", info.InstalledVersion, "to", info.AvailableVersion)
 			var err error
 			if bm != nil {
-				err = bm.UpgradeBackend(ctx, name, nil)
+				// Background auto-upgrade: no live admin watching a progress bar,
+				// so opID is empty and the distributed path skips progress streaming.
+				err = bm.UpgradeBackend(ctx, "", name, nil)
 			} else {
 				err = gallery.UpgradeBackend(ctx, uc.systemState, uc.modelLoader,
 					uc.galleries, name, nil, uc.appConfig.RequireBackendIntegrity)

core/backend/embeddings.go

@@ -100,8 +100,13 @@ func ModelEmbedding(ctx context.Context, s string, tokens []int, loader *model.M
 		trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes)
 
 		traceData := map[string]any{
-			"input_text":         trace.TruncateString(s, 1000),
-			"input_tokens_count": len(tokens),
+			"input_text": trace.TruncateString(s, 1000),
+		}
+		// Only present for token-mode callers (pre-tokenized override);
+		// emitting "0" alongside input_text would read as "consumed zero
+		// tokens", which is wrong.
+		if len(tokens) > 0 {
+			traceData["input_tokens_count"] = len(tokens)
 		}
 
 		startTime := time.Now()

core/backend/options.go

@@ -87,11 +87,47 @@ func getSeed(c config.ModelConfig) int32 {
 	return seed
 }
 
-func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions {
-	b := 512
-	if c.Batch != 0 {
-		b = c.Batch
+// DefaultContextSize and DefaultBatchSize are the backend's fallbacks when a
+// model config leaves them unset. Exported so callers that must respect the
+// effective decode window — notably the router's prompt trimmer — resolve the
+// same numbers grpcModelOpts does instead of guessing.
+const (
+	DefaultContextSize = 4096
+	DefaultBatchSize   = 512
+)
+
+// EffectiveContextSize is the context window the backend will run with: the
+// configured value, or DefaultContextSize when unset.
+func EffectiveContextSize(c config.ModelConfig) int {
+	if c.ContextSize != nil {
+		return *c.ContextSize
 	}
+	return DefaultContextSize
+}
+
+// EffectiveBatchSize is the single-decode batch the backend will run with.
+// Score, embedding and rerank all process the whole input in one pass: score
+// decodes prompt+candidate (asserts n_tokens <= n_batch), and embedding/rerank
+// pool over the full sequence in one physical batch (n_ubatch). So the batch
+// is sized to the context — anything that fits the context fits one pass,
+// avoiding both the GGML_ASSERT crash and the "input is too large to process"
+// error. Explicit `batch:` always wins.
+func EffectiveBatchSize(c config.ModelConfig) int {
+	if c.Batch != 0 {
+		return c.Batch
+	}
+	singlePass := c.HasUsecases(config.FLAG_SCORE) ||
+		c.HasUsecases(config.FLAG_EMBEDDINGS) ||
+		c.HasUsecases(config.FLAG_RERANK)
+	if ctx := EffectiveContextSize(c); singlePass && ctx > DefaultBatchSize {
+		return ctx
+	}
+	return DefaultBatchSize
+}
+
+func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions {
+	ctxSize := EffectiveContextSize(c)
+	b := EffectiveBatchSize(c)
 
 	flashAttention := "auto"
 
@@ -134,11 +170,6 @@ func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions {
 		}
 	}
 
-	ctxSize := 4096
-	if c.ContextSize != nil {
-		ctxSize = *c.ContextSize
-	}
-
 	mmlock := false
 	if c.MMlock != nil {
 		mmlock = *c.MMlock

core/backend/options_internal_test.go

@@ -97,3 +97,67 @@ var _ = Describe("gRPCPredictOpts reasoning_effort metadata", func() {
 		Expect(opts.Metadata).ToNot(HaveKey("reasoning_effort"))
 	})
 })
+
+var _ = Describe("grpcModelOpts NBatch", func() {
+	scoreUsecase := config.FLAG_SCORE
+	threads := 1
+	ctx := 4096
+
+	It("defaults to 512 for an ordinary model", func() {
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
+		opts := grpcModelOpts(cfg, "/tmp/models")
+		Expect(opts.NBatch).To(BeEquivalentTo(512))
+	})
+
+	It("sizes the batch to the context window for score models", func() {
+		// Score models decode the whole prompt+candidate in one
+		// llama_decode; n_batch must cover it or the backend aborts.
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}, KnownUsecases: &scoreUsecase}
+		opts := grpcModelOpts(cfg, "/tmp/models")
+		Expect(opts.NBatch).To(BeEquivalentTo(4096))
+	})
+
+	It("keeps an explicit batch over the score default", func() {
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}, KnownUsecases: &scoreUsecase}
+		cfg.Batch = 1024
+		opts := grpcModelOpts(cfg, "/tmp/models")
+		Expect(opts.NBatch).To(BeEquivalentTo(1024))
+	})
+
+	It("sizes the batch to the context window for embedding models", func() {
+		// Embedding/rerank pool over the whole sequence in one physical batch
+		// (n_ubatch); without this the input is capped at the 512 default and
+		// the backend returns "input is too large to process".
+		embeddings := true
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
+		cfg.Embeddings = &embeddings
+		opts := grpcModelOpts(cfg, "/tmp/models")
+		Expect(opts.NBatch).To(BeEquivalentTo(4096))
+	})
+
+	It("sizes the batch to the context window for rerank models", func() {
+		reranking := true
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
+		cfg.Reranking = &reranking
+		opts := grpcModelOpts(cfg, "/tmp/models")
+		Expect(opts.NBatch).To(BeEquivalentTo(4096))
+	})
+
+	It("does not raise the batch when a score model's context is below the default", func() {
+		small := 256
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &small}, KnownUsecases: &scoreUsecase}
+		opts := grpcModelOpts(cfg, "/tmp/models")
+		Expect(opts.NBatch).To(BeEquivalentTo(512))
+	})
+
+	It("sizes the batch to the effective 4096 default for a score model with no explicit context_size", func() {
+		// The crash case: the backend defaults n_ctx to 4096, so n_batch must
+		// follow even when context_size is unset — otherwise n_batch stays 512
+		// against a 4096 window and the score decode hits the GGML_ASSERT.
+		cfg := config.ModelConfig{Threads: &threads, KnownUsecases: &scoreUsecase}
+		Expect(cfg.ContextSize).To(BeNil())
+		opts := grpcModelOpts(cfg, "/tmp/models")
+		Expect(opts.NBatch).To(BeEquivalentTo(4096))
+		Expect(opts.ContextSize).To(BeEquivalentTo(4096), "n_batch must match the effective n_ctx the backend receives")
+	})
+})

core/backend/stores.go

@@ -3,9 +3,10 @@ package backend
 import (
 	"context"
 	"fmt"
-	"strings"
+	"time"
 
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/trace"
 
 	"github.com/mudler/LocalAI/pkg/grpc"
 	"github.com/mudler/LocalAI/pkg/model"
@@ -39,34 +40,85 @@ func (s *localVectorStore) backend(_ context.Context) (grpc.Backend, error) {
 	return StoreBackend(s.loader, s.appConfig, s.storeName, "")
 }
 
-func (s *localVectorStore) Search(ctx context.Context, vec []float32) (float64, []byte, bool, error) {
-	be, err := s.backend(ctx)
-	if err != nil {
-		return 0, nil, false, fmt.Errorf("vector store load: %w", err)
+func (s *localVectorStore) Search(ctx context.Context, vec []float32) (sim float64, payload []byte, ok bool, err error) {
+	start := time.Now()
+	outcome := "hit"
+	defer func() {
+		s.recordTrace(start, "search", len(vec), sim, outcome, err)
+	}()
+	be, berr := s.backend(ctx)
+	if berr != nil {
+		outcome = "backend_load_error"
+		return 0, nil, false, fmt.Errorf("vector store load: %w", berr)
 	}
-	_, values, similarities, err := store.Find(ctx, be, vec, 1)
-	if err != nil {
-		// local-store's Find returns "existing length is -1" before
-		// any keys are inserted. Surface that as a clean miss so the
-		// cache layer treats it as an empty store and proceeds to
-		// Insert rather than skipping.
-		if strings.Contains(err.Error(), "existing length is -1") {
-			return 0, nil, false, nil
-		}
-		return 0, nil, false, fmt.Errorf("vector store find: %w", err)
+	_, values, similarities, ferr := store.Find(ctx, be, vec, 1)
+	if ferr != nil {
+		outcome = "find_error"
+		return 0, nil, false, fmt.Errorf("vector store find: %w", ferr)
 	}
 	if len(values) == 0 || len(similarities) == 0 {
+		outcome = "miss"
 		return 0, nil, false, nil
 	}
 	return float64(similarities[0]), values[0], true, nil
 }
 
-func (s *localVectorStore) Insert(ctx context.Context, vec []float32, payload []byte) error {
-	be, err := s.backend(ctx)
-	if err != nil {
-		return fmt.Errorf("vector store load: %w", err)
+func (s *localVectorStore) Insert(ctx context.Context, vec []float32, payload []byte) (err error) {
+	start := time.Now()
+	outcome := "ok"
+	defer func() {
+		s.recordTrace(start, "insert", len(vec), 0, outcome, err)
+	}()
+	be, berr := s.backend(ctx)
+	if berr != nil {
+		outcome = "backend_load_error"
+		return fmt.Errorf("vector store load: %w", berr)
 	}
-	return store.SetSingle(ctx, be, vec, payload)
+	if serr := store.SetSingle(ctx, be, vec, payload); serr != nil {
+		outcome = "insert_error"
+		return serr
+	}
+	return nil
+}
+
+// recordTrace surfaces vector-store calls in /api/backend-traces, including
+// the backend-load-failure path that otherwise vanishes into an xlog.Warn.
+// modelName uses the store namespace (e.g. "router-cache-smart-router") so
+// admins can tell which router's cache misbehaved; the backend is always
+// "local-store" and can't disambiguate.
+func (s *localVectorStore) recordTrace(start time.Time, op string, vecDim int, sim float64, outcome string, err error) {
+	if s.appConfig == nil || !s.appConfig.EnableTracing {
+		return
+	}
+	trace.InitBackendTracingIfEnabled(s.appConfig.TracingMaxItems, s.appConfig.TracingMaxBodyBytes)
+	errStr := ""
+	if err != nil {
+		errStr = err.Error()
+	}
+	summary := op + " " + outcome
+	if op == "search" && outcome == "hit" {
+		summary = fmt.Sprintf("search hit (sim=%.3f)", sim)
+	}
+	data := map[string]any{
+		"op":         op,
+		"outcome":    outcome,
+		"vector_dim": vecDim,
+	}
+	// Only include similarity for a real neighbor — miss/empty_store would
+	// otherwise render "similarity: 0" and read as a measured value.
+	if op == "search" && outcome == "hit" {
+		data["similarity"] = sim
+	}
+	trace.RecordBackendTrace(trace.BackendTrace{
+		Timestamp: start,
+		Duration:  time.Since(start),
+		Type:      trace.BackendTraceVectorStore,
+		ModelName: s.storeName,
+		Backend:   model.LocalStoreBackend,
+		Summary:   summary,
+		Error:     errStr,
+		Data:      data,
+	})
 }
 
 func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, storeName string, backend string) (grpc.Backend, error) {

core/backend/stores_test.go

@@ -0,0 +1,88 @@
+package backend
+
+import (
+	"context"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/trace"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// findVectorStoreTrace returns the most recent vector_store trace whose
+// model_name matches storeName, or nil if none was recorded. Used by
+// the specs below to assert the trace landed without relying on
+// ring-buffer ordering across other tests in the suite.
+func findVectorStoreTrace(storeName string) *trace.BackendTrace {
+	traces := trace.GetBackendTraces()
+	for i := range traces {
+		bt := &traces[i]
+		if bt.Type == trace.BackendTraceVectorStore && bt.ModelName == storeName {
+			return bt
+		}
+	}
+	return nil
+}
+
+var _ = Describe("localVectorStore tracing", func() {
+	// Pin the trace surface admins read from /api/backend-traces.
+	// The original failure mode that motivated these specs — the
+	// local-store backend not installed — was silent on every surface
+	// except a per-call xlog.Warn. With tracing wired in, the row
+	// appears next to the embedder/score traces for the same request.
+	BeforeEach(func() {
+		trace.ClearBackendTraces()
+	})
+
+	It("records a vector_store trace with outcome=backend_load_error when the backend can't be loaded", func() {
+		// nil ModelLoader → s.backend → StoreBackend → panics on load.
+		// Use a real-but-empty loader so the failure surfaces as an
+		// error instead, exercising the load-failure trace path the
+		// admin would hit when local-store isn't installed.
+		appCfg := &config.ApplicationConfig{
+			EnableTracing:       true,
+			TracingMaxItems:     16,
+			TracingMaxBodyBytes: 1024,
+		}
+		s := &localVectorStore{
+			loader:    model.NewModelLoader(&system.SystemState{}),
+			appConfig: appCfg,
+			storeName: "router-cache-test",
+		}
+
+		// Search must surface the error AND record a trace describing it.
+		_, _, _, err := s.Search(context.Background(), []float32{0.1, 0.2, 0.3})
+		Expect(err).To(HaveOccurred())
+
+		Eventually(func() *trace.BackendTrace {
+			return findVectorStoreTrace("router-cache-test")
+		}).ShouldNot(BeNil())
+
+		bt := findVectorStoreTrace("router-cache-test")
+		Expect(bt.Backend).To(Equal(model.LocalStoreBackend))
+		Expect(bt.Data["op"]).To(Equal("search"))
+		Expect(bt.Data["outcome"]).To(Equal("backend_load_error"))
+		Expect(bt.Data["vector_dim"]).To(Equal(3))
+		// Error is the wrapped "vector store load: …" surfaced to the caller.
+		Expect(bt.Error).To(ContainSubstring("vector store load"))
+	})
+
+	It("does not record a trace when tracing is disabled", func() {
+		// Opt-out path: appConfig.EnableTracing=false must short-circuit
+		// before InitBackendTracingIfEnabled, so a workload with tracing
+		// turned off doesn't pay the channel-send cost per cache call.
+		appCfg := &config.ApplicationConfig{EnableTracing: false}
+		s := &localVectorStore{
+			loader:    model.NewModelLoader(&system.SystemState{}),
+			appConfig: appCfg,
+			storeName: "router-cache-disabled",
+		}
+		_, _, _, _ = s.Search(context.Background(), []float32{1})
+		Consistently(func() *trace.BackendTrace {
+			return findVectorStoreTrace("router-cache-disabled")
+		}).Should(BeNil())
+	})
+})

core/backend/tokenize.go

@@ -7,9 +7,23 @@ import (
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/mudler/LocalAI/core/trace"
 	"github.com/mudler/LocalAI/pkg/grpc"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/model"
 )
 
+// tokenizeTokenCount returns the number of tokens in a backend response,
+// treating a nil response as zero. The gRPC client returns (nil, err) on
+// failure, and the tracing block below runs before that error is returned —
+// so the count must be read nil-safely here. Reading resp.Tokens on a nil
+// resp previously panicked the whole HTTP handler when tracing was enabled
+// (e.g. a transient tokenize failure during router probe-budget sizing).
+func tokenizeTokenCount(resp *pb.TokenizationResponse) int {
+	if resp == nil {
+		return 0
+	}
+	return len(resp.Tokens)
+}
+
 func ModelTokenize(s string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (schema.TokenizeResponse, error) {
 
 	var inferenceModel grpc.Backend
@@ -40,10 +54,7 @@ func ModelTokenize(s string, loader *model.ModelLoader, modelConfig config.Model
 			errStr = err.Error()
 		}
 
-		tokenCount := 0
-		if resp.Tokens != nil {
-			tokenCount = len(resp.Tokens)
-		}
+		tokenCount := tokenizeTokenCount(resp)
 
 		trace.RecordBackendTrace(trace.BackendTrace{
 			Timestamp: startTime,
@@ -64,8 +75,8 @@ func ModelTokenize(s string, loader *model.ModelLoader, modelConfig config.Model
 		return schema.TokenizeResponse{}, err
 	}
 
-	if resp.Tokens == nil {
-		resp.Tokens = make([]int32, 0)
+	if resp == nil || resp.Tokens == nil {
+		return schema.TokenizeResponse{Tokens: make([]int32, 0)}, nil
 	}
 
 	return schema.TokenizeResponse{

core/backend/tokenize_test.go

@@ -0,0 +1,27 @@
+package backend
+
+import (
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("tokenizeTokenCount", func() {
+	// Regression: the gRPC client returns (nil, err) when a tokenize call
+	// fails, and ModelTokenize's tracing block reads the token count before
+	// the error is returned. Dereferencing a nil response there panicked the
+	// HTTP handler (nil pointer dereference) — e.g. a transient tokenize
+	// failure while the router sized its probe-token budget.
+	It("returns zero for a nil response instead of panicking", func() {
+		Expect(tokenizeTokenCount(nil)).To(Equal(0))
+	})
+
+	It("returns zero when the response carries no tokens", func() {
+		Expect(tokenizeTokenCount(&pb.TokenizationResponse{})).To(Equal(0))
+	})
+
+	It("counts the tokens present on the response", func() {
+		Expect(tokenizeTokenCount(&pb.TokenizationResponse{Tokens: []int32{1, 2, 3}})).To(Equal(3))
+	})
+})

core/cli/chat/chat.go

@@ -0,0 +1,30 @@
+package chat
+
+import (
+	"context"
+	"io"
+	"strings"
+)
+
+type Options struct {
+	Model   string
+	BaseURL string
+	APIKey  string
+	In      io.Reader
+	Out     io.Writer
+}
+
+func Run(ctx context.Context, opts Options) error {
+	if opts.In == nil {
+		opts.In = strings.NewReader("")
+	}
+	if opts.Out == nil {
+		opts.Out = io.Discard
+	}
+
+	session, err := newChatSession(ctx, newLocalAIChatClient(opts.BaseURL, opts.APIKey), opts.Model)
+	if err != nil {
+		return err
+	}
+	return runTerminalChat(ctx, session, opts.In, opts.Out)
+}

core/cli/chat/chat_suite_test.go

@@ -0,0 +1,13 @@
+package chat
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestChat(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Chat Suite")
+}

core/cli/chat/chat_test.go

@@ -0,0 +1,172 @@
+package chat
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Run chat", func() {
+	It("streams a single chat response", func() {
+		var capturedModel string
+		var capturedAuth string
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == "/v1/models" {
+				w.Header().Set("Content-Type", "application/json")
+				writeResponse(w, `{"object":"list","data":[{"id":"test-model","object":"model"}]}`)
+				return
+			}
+
+			Expect(r.URL.Path).To(Equal("/v1/chat/completions"))
+			capturedAuth = r.Header.Get("Authorization")
+
+			var body struct {
+				Model    string `json:"model"`
+				Messages []struct {
+					Role    string `json:"role"`
+					Content string `json:"content"`
+				} `json:"messages"`
+			}
+			Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed())
+			capturedModel = body.Model
+			Expect(body.Messages).To(HaveLen(1))
+			Expect(body.Messages[0].Role).To(Equal("user"))
+			Expect(body.Messages[0].Content).To(Equal("hello"))
+
+			w.Header().Set("Content-Type", "text/event-stream")
+			writeResponse(w, "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}]}\n\n")
+			writeResponse(w, "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"}}]}\n\n")
+			writeResponse(w, "data: [DONE]\n\n")
+		}))
+		defer server.Close()
+
+		var out bytes.Buffer
+		err := Run(GinkgoT().Context(), Options{
+			Model:   "test-model",
+			BaseURL: server.URL + "/v1",
+			APIKey:  "secret",
+			In:      strings.NewReader("hello\n/exit\n"),
+			Out:     &out,
+		})
+
+		Expect(err).ToNot(HaveOccurred())
+		Expect(capturedModel).To(Equal("test-model"))
+		Expect(capturedAuth).To(Equal("Bearer secret"))
+		Expect(out.String()).To(ContainSubstring("assistant: hi!"))
+		Expect(out.String()).To(ContainSubstring("bye"))
+	})
+
+	It("auto-selects the only available model", func() {
+		server := chatTestServer([]string{"solo"}, nil)
+		defer server.Close()
+
+		var out bytes.Buffer
+		err := Run(GinkgoT().Context(), Options{
+			BaseURL: server.URL + "/v1",
+			In:      strings.NewReader("/exit\n"),
+			Out:     &out,
+		})
+
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out.String()).To(ContainSubstring("LocalAI chat (solo)"))
+	})
+
+	It("returns an actionable error when no models are installed", func() {
+		server := chatTestServer(nil, nil)
+		defer server.Close()
+
+		err := Run(GinkgoT().Context(), Options{
+			BaseURL: server.URL + "/v1",
+			In:      strings.NewReader(""),
+		})
+
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("no chat models are installed"))
+		Expect(err.Error()).To(ContainSubstring("local-ai models install <model>"))
+	})
+
+	It("returns an actionable error when multiple models are available without a selection", func() {
+		server := chatTestServer([]string{"alpha", "beta"}, nil)
+		defer server.Close()
+
+		err := Run(GinkgoT().Context(), Options{
+			BaseURL: server.URL + "/v1",
+			In:      strings.NewReader(""),
+		})
+
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("multiple models are available"))
+		Expect(err.Error()).To(ContainSubstring("--model"))
+		Expect(err.Error()).To(ContainSubstring("alpha"))
+		Expect(err.Error()).To(ContainSubstring("beta"))
+	})
+
+	It("lists and switches models inside the chat", func() {
+		requestedModels := []string{}
+		server := chatTestServer([]string{"alpha", "beta"}, func(model string) {
+			requestedModels = append(requestedModels, model)
+		})
+		defer server.Close()
+
+		var out bytes.Buffer
+		err := Run(GinkgoT().Context(), Options{
+			Model:   "alpha",
+			BaseURL: server.URL + "/v1",
+			In:      strings.NewReader("/models\n/model beta\nhello\n/exit\n"),
+			Out:     &out,
+		})
+
+		Expect(err).ToNot(HaveOccurred())
+		Expect(out.String()).To(ContainSubstring("* alpha"))
+		Expect(out.String()).To(ContainSubstring("  beta"))
+		Expect(out.String()).To(ContainSubstring("switched to beta; conversation cleared"))
+		Expect(requestedModels).To(Equal([]string{"beta"}))
+	})
+})
+
+func chatTestServer(models []string, onChat func(model string)) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v1/models":
+			w.Header().Set("Content-Type", "application/json")
+			writeResponse(w, `{"object":"list","data":[`)
+			for i, model := range models {
+				if i > 0 {
+					writeResponse(w, ",")
+				}
+				writeResponsef(w, `{"id":%q,"object":"model"}`, model)
+			}
+			writeResponse(w, `]}`)
+		case "/v1/chat/completions":
+			var body struct {
+				Model string `json:"model"`
+			}
+			Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed())
+			if onChat != nil {
+				onChat(body.Model)
+			}
+			w.Header().Set("Content-Type", "text/event-stream")
+			writeResponse(w, "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"}}]}\n\n")
+			writeResponse(w, "data: [DONE]\n\n")
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+}
+
+func writeResponse(w io.Writer, text string) {
+	_, err := fmt.Fprint(w, text)
+	Expect(err).ToNot(HaveOccurred())
+}
+
+func writeResponsef(w io.Writer, format string, args ...any) {
+	_, err := fmt.Fprintf(w, format, args...)
+	Expect(err).ToNot(HaveOccurred())
+}

core/cli/chat/client.go

@@ -0,0 +1,114 @@
+package chat
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	openai "github.com/sashabaranov/go-openai"
+)
+
+type chatClient interface {
+	ListModels(ctx context.Context) ([]string, error)
+	StreamChat(ctx context.Context, model string, messages []chatMessage, out io.Writer) (string, error)
+}
+
+type localAIChatClient struct {
+	client *openai.Client
+}
+
+func newLocalAIChatClient(baseURL string, apiKey string) *localAIChatClient {
+	cfg := openai.DefaultConfig(apiKey)
+	cfg.BaseURL = baseURL
+	return &localAIChatClient{client: openai.NewClientWithConfig(cfg)}
+}
+
+func (c *localAIChatClient) ListModels(ctx context.Context) ([]string, error) {
+	resp, err := c.client.ListModels(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	models := make([]string, 0, len(resp.Models))
+	for _, model := range resp.Models {
+		if model.ID != "" {
+			models = append(models, model.ID)
+		}
+	}
+	sort.Strings(models)
+	return models, nil
+}
+
+func (c *localAIChatClient) StreamChat(ctx context.Context, model string, messages []chatMessage, out io.Writer) (string, error) {
+	stream, err := c.client.CreateChatCompletionStream(ctx, openai.ChatCompletionRequest{
+		Model:    model,
+		Messages: openAIChatMessages(messages),
+	})
+	if err != nil {
+		return "", friendlyChatError(err, model)
+	}
+	defer func() {
+		_ = stream.Close()
+	}()
+
+	var answer strings.Builder
+	for {
+		resp, err := stream.Recv()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return answer.String(), friendlyChatError(err, model)
+		}
+		if len(resp.Choices) == 0 {
+			continue
+		}
+
+		token := resp.Choices[0].Delta.Content
+		if token == "" {
+			continue
+		}
+		answer.WriteString(token)
+		if _, err := fmt.Fprint(out, token); err != nil {
+			return answer.String(), err
+		}
+	}
+
+	return answer.String(), nil
+}
+
+func openAIChatMessages(messages []chatMessage) []openai.ChatCompletionMessage {
+	converted := make([]openai.ChatCompletionMessage, len(messages))
+	for i, message := range messages {
+		converted[i] = openai.ChatCompletionMessage{
+			Role:    message.Role,
+			Content: message.Content,
+		}
+	}
+	return converted
+}
+
+func friendlyChatError(err error, model string) error {
+	var apiErr *openai.APIError
+	if errors.As(err, &apiErr) {
+		switch apiErr.HTTPStatusCode {
+		case 404:
+			return fmt.Errorf("model %q is not available. Run `local-ai models list`, install a model with `local-ai models install <model>`, or switch with `/model <name>`", model)
+		case 403:
+			return fmt.Errorf("model %q is disabled. Enable it from LocalAI settings or choose another model with `/model <name>`", model)
+		}
+		if apiErr.Message != "" {
+			return errors.New(apiErr.Message)
+		}
+	}
+
+	msg := err.Error()
+	if strings.Contains(msg, "model") && strings.Contains(msg, "not found") {
+		return fmt.Errorf("model %q is not available. Run `local-ai models list`, install a model with `local-ai models install <model>`, or switch with `/model <name>`", model)
+	}
+
+	return err
+}

core/cli/chat/models.go

@@ -0,0 +1,17 @@
+package chat
+
+import "strings"
+
+func formatChatModelList(models []string, current string) string {
+	var b strings.Builder
+	for _, model := range models {
+		prefix := "  "
+		if model == current {
+			prefix = "* "
+		}
+		b.WriteString(prefix)
+		b.WriteString(model)
+		b.WriteByte('\n')
+	}
+	return b.String()
+}

core/cli/chat/session.go

@@ -0,0 +1,120 @@
+package chat
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+)
+
+const (
+	chatRoleUser      = "user"
+	chatRoleAssistant = "assistant"
+)
+
+type chatMessage struct {
+	Role    string
+	Content string
+}
+
+type chatSession struct {
+	client   chatClient
+	model    string
+	models   []string
+	messages []chatMessage
+}
+
+func newChatSession(ctx context.Context, client chatClient, requestedModel string) (*chatSession, error) {
+	models, err := client.ListModels(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("list models: %w", err)
+	}
+
+	model, err := resolveChatModel(requestedModel, models)
+	if err != nil {
+		return nil, err
+	}
+
+	return &chatSession{
+		client: client,
+		model:  model,
+		models: models,
+	}, nil
+}
+
+func (s *chatSession) CurrentModel() string {
+	return s.model
+}
+
+func (s *chatSession) Models() []string {
+	models := make([]string, len(s.models))
+	copy(models, s.models)
+	return models
+}
+
+func (s *chatSession) Clear() {
+	s.messages = nil
+}
+
+func (s *chatSession) SwitchModel(model string) error {
+	if !modelExists(s.models, model) {
+		return fmt.Errorf("model %q is not available. Use /models to see installed models", model)
+	}
+	s.model = model
+	s.Clear()
+	return nil
+}
+
+func (s *chatSession) Send(ctx context.Context, prompt string, out io.Writer) error {
+	s.messages = append(s.messages, chatMessage{
+		Role:    chatRoleUser,
+		Content: prompt,
+	})
+
+	answer, err := s.client.StreamChat(ctx, s.model, s.messages, out)
+	if err != nil {
+		return err
+	}
+
+	s.messages = append(s.messages, chatMessage{
+		Role:    chatRoleAssistant,
+		Content: answer,
+	})
+	return nil
+}
+
+func resolveChatModel(requested string, models []string) (string, error) {
+	switch {
+	case requested == "" && len(models) == 0:
+		return "", errors.New(`no chat models are installed.
+
+Install a model first, for example:
+  local-ai models list
+  local-ai models install <model>
+  local-ai run
+
+Then start a chat session:
+  local-ai chat --model <model>`)
+	case requested == "" && len(models) == 1:
+		return models[0], nil
+	case requested == "" && len(models) > 1:
+		var b strings.Builder
+		b.WriteString("multiple models are available; choose one with --model:\n")
+		b.WriteString(formatChatModelList(models, ""))
+		return "", errors.New(b.String())
+	case !modelExists(models, requested):
+		return "", fmt.Errorf("model %q is not available. Use `local-ai models list` and `local-ai models install <model>`, or pass an installed model with --model", requested)
+	default:
+		return requested, nil
+	}
+}
+
+func modelExists(models []string, name string) bool {
+	for _, model := range models {
+		if model == name {
+			return true
+		}
+	}
+	return false
+}

core/cli/chat/session_test.go

@@ -0,0 +1,56 @@
+package chat
+
+import (
+	"context"
+	"io"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Chat session", func() {
+	It("keeps model switching and message history out of the terminal adapter", func() {
+		client := &fakeChatClient{
+			models: []string{"alpha", "beta"},
+			answer: "pong",
+		}
+
+		session, err := newChatSession(context.Background(), client, "alpha")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(session.CurrentModel()).To(Equal("alpha"))
+
+		Expect(session.SwitchModel("beta")).To(Succeed())
+		Expect(session.CurrentModel()).To(Equal("beta"))
+		Expect(session.Send(context.Background(), "ping", io.Discard)).To(Succeed())
+
+		Expect(client.requests).To(HaveLen(1))
+		Expect(client.requests[0].model).To(Equal("beta"))
+		Expect(client.requests[0].messages).To(HaveLen(1))
+		Expect(client.requests[0].messages[0].Content).To(Equal("ping"))
+	})
+})
+
+type fakeChatClient struct {
+	models   []string
+	answer   string
+	requests []fakeChatRequest
+}
+
+type fakeChatRequest struct {
+	model    string
+	messages []chatMessage
+}
+
+func (c *fakeChatClient) ListModels(context.Context) ([]string, error) {
+	return c.models, nil
+}
+
+func (c *fakeChatClient) StreamChat(_ context.Context, model string, messages []chatMessage, out io.Writer) (string, error) {
+	copied := make([]chatMessage, len(messages))
+	copy(copied, messages)
+	c.requests = append(c.requests, fakeChatRequest{model: model, messages: copied})
+	if _, err := io.WriteString(out, c.answer); err != nil {
+		return "", err
+	}
+	return c.answer, nil
+}

core/cli/chat/terminal.go

@@ -0,0 +1,93 @@
+package chat
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+)
+
+func runTerminalChat(ctx context.Context, session *chatSession, in io.Reader, out io.Writer) error {
+	scanner := bufio.NewScanner(in)
+	scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
+
+	if err := writeChat(out, "LocalAI chat (%s)\n", session.CurrentModel()); err != nil {
+		return err
+	}
+	if err := writeChat(out, "Type /exit to quit, /clear to reset the conversation, /models to list models.\n"); err != nil {
+		return err
+	}
+
+	for {
+		if err := writeChat(out, "\n> "); err != nil {
+			return err
+		}
+		if !scanner.Scan() {
+			break
+		}
+
+		prompt := strings.TrimSpace(scanner.Text())
+		switch prompt {
+		case "":
+			continue
+		case "/bye", "/exit", "/quit":
+			return writeChat(out, "bye\n")
+		case "/clear":
+			session.Clear()
+			if err := writeChat(out, "conversation cleared\n"); err != nil {
+				return err
+			}
+			continue
+		case "/models":
+			if err := printChatModels(out, session.Models(), session.CurrentModel()); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if nextModel, ok := strings.CutPrefix(prompt, "/model "); ok {
+			nextModel = strings.TrimSpace(nextModel)
+			if nextModel == "" {
+				if err := writeChat(out, "usage: /model <name>\n"); err != nil {
+					return err
+				}
+				continue
+			}
+			if err := session.SwitchModel(nextModel); err != nil {
+				if writeErr := writeChat(out, "%s\n", err); writeErr != nil {
+					return writeErr
+				}
+				continue
+			}
+			if err := writeChat(out, "switched to %s; conversation cleared\n", session.CurrentModel()); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if err := writeChat(out, "assistant: "); err != nil {
+			return err
+		}
+		if err := session.Send(ctx, prompt, out); err != nil {
+			return err
+		}
+		if err := writeChat(out, "\n"); err != nil {
+			return err
+		}
+	}
+
+	return scanner.Err()
+}
+
+func printChatModels(out io.Writer, models []string, current string) error {
+	if len(models) == 0 {
+		return writeChat(out, "no models installed\n")
+	}
+	return writeChat(out, "%s", formatChatModelList(models, current))
+}
+
+func writeChat(out io.Writer, format string, args ...any) error {
+	_, err := fmt.Fprintf(out, format, args...)
+	return err
+}

core/cli/chat_cmd.go

@@ -0,0 +1,25 @@
+package cli
+
+import (
+	"context"
+	"os"
+
+	chatcli "github.com/mudler/LocalAI/core/cli/chat"
+	cliContext "github.com/mudler/LocalAI/core/cli/context"
+)
+
+type ChatCMD struct {
+	Model    string `short:"m" help:"Model name to use. Defaults to the only model returned by the server when exactly one is available"`
+	Endpoint string `env:"LOCALAI_CHAT_ENDPOINT" default:"http://127.0.0.1:8080" help:"LocalAI server endpoint. The /v1 path is added automatically when omitted"`
+	APIKey   string `env:"LOCALAI_API_KEY,API_KEY" help:"API key to use when the LocalAI server requires authentication"`
+}
+
+func (c *ChatCMD) Run(ctx *cliContext.Context) error {
+	return chatcli.Run(context.Background(), chatcli.Options{
+		Model:   c.Model,
+		BaseURL: chatAPIBaseURL(c.Endpoint),
+		APIKey:  c.APIKey,
+		In:      os.Stdin,
+		Out:     os.Stdout,
+	})
+}

core/cli/chat_cmd_test.go

@@ -0,0 +1,27 @@
+package cli
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Chat command wiring", func() {
+	Describe("chatAPIBaseURL", func() {
+		It("adds /v1 to a root endpoint", func() {
+			Expect(chatAPIBaseURL("http://127.0.0.1:8080")).To(Equal("http://127.0.0.1:8080/v1"))
+		})
+
+		It("keeps endpoints that already include /v1", func() {
+			Expect(chatAPIBaseURL("http://127.0.0.1:8080/v1")).To(Equal("http://127.0.0.1:8080/v1"))
+			Expect(chatAPIBaseURL("http://127.0.0.1:8080/v1/")).To(Equal("http://127.0.0.1:8080/v1"))
+		})
+
+		It("adds a default http scheme", func() {
+			Expect(chatAPIBaseURL("127.0.0.1:8080")).To(Equal("http://127.0.0.1:8080/v1"))
+		})
+
+		It("preserves non-root paths before /v1", func() {
+			Expect(chatAPIBaseURL("http://127.0.0.1:8080/localai")).To(Equal("http://127.0.0.1:8080/localai/v1"))
+		})
+	})
+})

core/cli/chat_endpoint.go

@@ -0,0 +1,29 @@
+package cli
+
+import (
+	"net/url"
+	"strings"
+)
+
+func chatAPIBaseURL(endpoint string) string {
+	if !strings.Contains(endpoint, "://") {
+		endpoint = "http://" + endpoint
+	}
+
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return strings.TrimRight(endpoint, "/") + "/v1"
+	}
+
+	path := strings.TrimRight(u.Path, "/")
+	if path == "" {
+		u.Path = "/v1"
+	} else if path != "/v1" && !strings.HasSuffix(path, "/v1") {
+		u.Path = path + "/v1"
+	} else {
+		u.Path = path
+	}
+	u.RawQuery = ""
+	u.Fragment = ""
+	return u.String()
+}

core/cli/cli.go

@@ -9,6 +9,7 @@ var CLI struct {
 	cliContext.Context `embed:""`
 
 	Run             RunCMD             `cmd:"" help:"Run LocalAI, this the default command if no other command is specified. Run 'local-ai run --help' for more information" default:"withargs"`
+	Chat            ChatCMD            `cmd:"" help:"Open an interactive chat session against a running LocalAI server"`
 	Federated       FederatedCLI       `cmd:"" help:"Run LocalAI in federated mode"`
 	Models          ModelsCMD          `cmd:"" help:"Manage LocalAI models and definitions"`
 	Backends        BackendsCMD        `cmd:"" help:"Manage LocalAI backends and definitions"`

core/cli/run.go

@@ -30,6 +30,8 @@ type RunCMD struct {
 	ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"`
 
 	ExternalBackends             []string      `env:"LOCALAI_EXTERNAL_BACKENDS,EXTERNAL_BACKENDS" help:"A list of external backends to load from gallery on boot" group:"backends"`
+	WebRTCNAT1To1IPs             []string      `env:"LOCALAI_WEBRTC_NAT_1TO1_IPS,WEBRTC_NAT_1TO1_IPS" help:"IPs advertised as the host ICE candidates for /v1/realtime WebRTC instead of every local interface. Set to the reachable host/LAN IP when running under Docker host networking or NAT, where pion otherwise offers unreachable bridge addresses and the connection drops after ICE consent checks fail." group:"api"`
+	WebRTCICEInterfaces          []string      `env:"LOCALAI_WEBRTC_ICE_INTERFACES,WEBRTC_ICE_INTERFACES" help:"Restrict /v1/realtime WebRTC ICE candidate gathering to these network interfaces (e.g. eth0), filtering out docker0/veth noise." group:"api"`
 	BackendsPath                 string        `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"`
 	BackendsSystemPath           string        `env:"LOCALAI_BACKENDS_SYSTEM_PATH,BACKEND_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends used for inferencing" group:"backends"`
 	ModelsPath                   string        `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
@@ -225,6 +227,8 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		config.WithApiKeys(r.APIKeys),
 		config.WithModelsURL(append(r.Models, r.ModelArgs...)...),
 		config.WithExternalBackends(r.ExternalBackends...),
+		config.WithWebRTCNAT1To1IPs(r.WebRTCNAT1To1IPs...),
+		config.WithWebRTCICEInterfaces(r.WebRTCICEInterfaces...),
 		config.WithOpaqueErrors(r.OpaqueErrors),
 		config.WithEnforcedPredownloadScans(!r.DisablePredownloadScan),
 		config.WithSubtleKeyComparison(r.UseSubtleKeyComparison),
@@ -652,12 +656,12 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 // waitForServerReady polls the given address until the HTTP server is
 // accepting connections or the context is cancelled.
 func waitForServerReady(address string, ctx context.Context) {
-	// Ensure the address has a host component for dialing.
-	// Echo accepts ":8080" but net.Dial needs a resolvable host.
 	host, port, err := net.SplitHostPort(address)
 	if err == nil && host == "" {
 		address = "127.0.0.1:" + port
 	}
+	ticker := time.NewTicker(250 * time.Millisecond)
+	defer ticker.Stop()
 
 	for {
 		select {
@@ -665,11 +669,17 @@ func waitForServerReady(address string, ctx context.Context) {
 			return
 		default:
 		}
+
 		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
 		if err == nil {
 			conn.Close()
 			return
 		}
-		time.Sleep(250 * time.Millisecond)
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
 	}
 }

core/config/application_config.go

@@ -12,10 +12,19 @@ import (
 )
 
 type ApplicationConfig struct {
-	Context                             context.Context
-	ConfigFile                          string
-	SystemState                         *system.SystemState
-	ExternalBackends                    []string
+	Context          context.Context
+	ConfigFile       string
+	SystemState      *system.SystemState
+	ExternalBackends []string
+
+	// WebRTCNAT1To1IPs, when set, are advertised as the host ICE candidates for
+	// /v1/realtime WebRTC instead of every local interface address. Needed when
+	// the routable address differs from what pion gathers — e.g. Docker host
+	// networking (where pion also offers unreachable bridge IPs) or NAT.
+	WebRTCNAT1To1IPs []string
+	// WebRTCICEInterfaces, when set, restricts ICE candidate gathering to these
+	// network interfaces (e.g. eth0), filtering out docker0/veth noise.
+	WebRTCICEInterfaces                 []string
 	UploadLimitMB, Threads, ContextSize int
 	F16                                 bool
 	Debug                               bool
@@ -56,7 +65,7 @@ type ApplicationConfig struct {
 	//
 	//   patterns:
 	//     - id: email
-	//       action: route_local      # downgrade default mask -> route_local
+	//       action: allow            # downgrade default mask -> allow (log only)
 	//     - id: ssn
 	//       action: block            # upgrade default mask -> block
 	//
@@ -81,7 +90,6 @@ type ApplicationConfig struct {
 	// file is mode 0600.
 	MITMCADir string
 
-
 	// PIIPatternOverrides applies persisted per-id deltas (action,
 	// disabled) to the live redactor at startup. Loaded from
 	// runtime_settings.json and applied right after pii.NewRedactor.
@@ -116,11 +124,11 @@ type ApplicationConfig struct {
 	// --require-backend-integrity / LOCALAI_REQUIRE_BACKEND_INTEGRITY.
 	RequireBackendIntegrity bool
 
-	SingleBackend           bool // Deprecated: use MaxActiveBackends = 1 instead
-	MaxActiveBackends       int  // Maximum number of active backends (0 = unlimited, 1 = single backend mode)
-	WatchDogIdle bool
-	WatchDogBusy bool
-	WatchDog     bool
+	SingleBackend     bool // Deprecated: use MaxActiveBackends = 1 instead
+	MaxActiveBackends int  // Maximum number of active backends (0 = unlimited, 1 = single backend mode)
+	WatchDogIdle      bool
+	WatchDogBusy      bool
+	WatchDog          bool
 
 	// Memory Reclaimer settings (works with GPU if available, otherwise RAM)
 	MemoryReclaimerEnabled   bool    // Enable memory threshold monitoring
@@ -311,6 +319,18 @@ func WithExternalBackends(backends ...string) AppOption {
 	}
 }
 
+func WithWebRTCNAT1To1IPs(ips ...string) AppOption {
+	return func(o *ApplicationConfig) {
+		o.WebRTCNAT1To1IPs = ips
+	}
+}
+
+func WithWebRTCICEInterfaces(interfaces ...string) AppOption {
+	return func(o *ApplicationConfig) {
+		o.WebRTCICEInterfaces = interfaces
+	}
+}
+
 func WithMachineTag(tag string) AppOption {
 	return func(o *ApplicationConfig) {
 		o.MachineTag = tag
@@ -702,7 +722,6 @@ func WithMITMCADir(dir string) AppOption {
 	}
 }
 
-
 func WithDynamicConfigDir(dynamicConfigsDir string) AppOption {
 	return func(o *ApplicationConfig) {
 		o.DynamicConfigsDir = dynamicConfigsDir

core/config/hooks_llamacpp.go

@@ -39,7 +39,21 @@ func llamaCppDefaults(cfg *ModelConfig, modelPath string) {
 		}
 	}()
 
-	f, err := gguf.ParseGGUFFile(guessPath)
+	// Startup parses every model's GGUF header to guess defaults. We only need
+	// scalar metadata (architecture, head/ff counts, chat_template, token IDs,
+	// MTP head) plus array *lengths* — never the array *contents*. Two options
+	// keep this cheap, which matters when many models live on slow storage such
+	// as a Docker volume (see https://github.com/mudler/LocalAI/issues/9790):
+	//
+	//   - SkipLargeMetadata: seek past large array-valued metadata (the tokenizer
+	//     vocab: tokenizer.ggml.tokens/scores/merges, often >100k entries) instead
+	//     of reading and allocating every element. Lengths stay populated.
+	//   - UseMMap: read the header via a memory map so faulting in a few pages
+	//     replaces hundreds of thousands of tiny read() syscalls (measured ~524k
+	//     -> 8 for a 256k-token vocab), the dominant cost on slow filesystems.
+	//
+	// The mapping is released when ParseGGUFFile returns.
+	f, err := gguf.ParseGGUFFile(guessPath, gguf.UseMMap(), gguf.SkipLargeMetadata())
 	if err == nil {
 		guessGGUFFromFile(cfg, f, 0)
 	}

core/config/hooks_test.go

@@ -1,13 +1,76 @@
 package config_test
 
 import (
+	"bytes"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/schema"
 
+	gguf "github.com/gpustack/gguf-parser-go"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
+// GGUF metadata value type tags (see github.com/gpustack/gguf-parser-go).
+const (
+	ggufTypeUint32 uint32 = 4
+	ggufTypeString uint32 = 8
+	ggufTypeArray  uint32 = 9
+)
+
+// writeTestGGUF emits a minimal but valid little-endian GGUF v3 header carrying
+// the scalar metadata the llama-cpp hook guesses from plus a large string vocab
+// array (tokenizer.ggml.tokens). The big array is exactly what SkipLargeMetadata
+// + UseMMap are expected to avoid reading element-by-element, so it must survive a
+// round-trip through the real hook without corrupting the guessed defaults.
+func writeTestGGUF(path, chatTemplate string, vocab int) error {
+	wStr := func(b *bytes.Buffer, s string) {
+		binary.Write(b, binary.LittleEndian, uint64(len(s)))
+		b.WriteString(s)
+	}
+	kvStr := func(b *bytes.Buffer, k, v string) {
+		wStr(b, k)
+		binary.Write(b, binary.LittleEndian, ggufTypeString)
+		wStr(b, v)
+	}
+	kvU32 := func(b *bytes.Buffer, k string, v uint32) {
+		wStr(b, k)
+		binary.Write(b, binary.LittleEndian, ggufTypeUint32)
+		binary.Write(b, binary.LittleEndian, v)
+	}
+
+	var meta bytes.Buffer
+	kvStr(&meta, "general.architecture", "llama")
+	kvStr(&meta, "general.name", "ReproModel")
+	kvU32(&meta, "llama.context_length", 4096)
+	kvU32(&meta, "llama.attention.head_count", 32)
+	kvU32(&meta, "llama.feed_forward_length", 11008)
+	kvU32(&meta, "llama.block_count", 32)
+	kvU32(&meta, "tokenizer.ggml.bos_token_id", 1)
+	kvStr(&meta, "tokenizer.chat_template", chatTemplate)
+
+	// large array value — the one the optimization skips reading
+	wStr(&meta, "tokenizer.ggml.tokens")
+	binary.Write(&meta, binary.LittleEndian, ggufTypeArray)
+	binary.Write(&meta, binary.LittleEndian, ggufTypeString)
+	binary.Write(&meta, binary.LittleEndian, uint64(vocab))
+	for i := 0; i < vocab; i++ {
+		wStr(&meta, "token")
+	}
+
+	var out bytes.Buffer
+	binary.Write(&out, binary.LittleEndian, gguf.GGUFMagicGGUFLe)
+	binary.Write(&out, binary.LittleEndian, uint32(3)) // version
+	binary.Write(&out, binary.LittleEndian, uint64(0)) // tensor count
+	binary.Write(&out, binary.LittleEndian, uint64(9)) // metadata kv count
+	out.Write(meta.Bytes())
+
+	return os.WriteFile(path, out.Bytes(), 0o644)
+}
+
 var _ = Describe("Backend hooks and parser defaults", func() {
 	Context("MatchParserDefaults", func() {
 		It("matches Qwen3 family", func() {
@@ -137,6 +200,58 @@ var _ = Describe("Backend hooks and parser defaults", func() {
 		})
 	})
 
+	Context("llamaCppDefaults GGUF guessing", func() {
+		// Regression coverage for https://github.com/mudler/LocalAI/issues/9790:
+		// the hook reads GGUF headers with SkipLargeMetadata + UseMMap to avoid
+		// pulling the whole tokenizer vocab off (slow) disk on every startup. This
+		// verifies that skipping the vocab array still yields the correct guessed
+		// defaults from the remaining scalar metadata.
+		const chatTemplate = "{{ bos_token }}{% for m in messages %}{{ m.content }}{% endfor %}"
+
+		It("guesses defaults from a GGUF whose large vocab is skipped", func() {
+			dir := GinkgoT().TempDir()
+			modelFile := "repro.gguf"
+			Expect(writeTestGGUF(filepath.Join(dir, modelFile), chatTemplate, 50000)).To(Succeed())
+
+			// A pre-set context size short-circuits the GGUF run-estimate, which
+			// needs full tensor info this header-only fixture deliberately omits;
+			// the metadata-reading path the optimization touches is unaffected.
+			ctxSize := 4096
+			cfg := &ModelConfig{
+				Backend: "llama-cpp",
+				LLMConfig: LLMConfig{ContextSize: &ctxSize},
+				PredictionOptions: schema.PredictionOptions{
+					BasicModelRequest: schema.BasicModelRequest{Model: modelFile},
+				},
+			}
+			cfg.SetDefaults(ModelPath(dir))
+
+			// chat_template is a scalar string, not part of the skipped array,
+			// so it must be captured verbatim.
+			Expect(cfg.GetModelTemplate()).To(Equal(chatTemplate))
+			// scalar-derived defaults are still applied
+			Expect(cfg.ContextSize).NotTo(BeNil())
+			Expect(cfg.NGPULayers).NotTo(BeNil())
+			Expect(cfg.TemplateConfig.UseTokenizerTemplate).To(BeTrue())
+			Expect(cfg.KnownUsecaseStrings).To(ContainElement("FLAG_CHAT"))
+		})
+
+		It("falls back to the default context size when the GGUF is unreadable", func() {
+			dir := GinkgoT().TempDir()
+			Expect(os.WriteFile(filepath.Join(dir, "bad.gguf"), []byte("not a gguf"), 0o644)).To(Succeed())
+
+			cfg := &ModelConfig{
+				Backend: "llama-cpp",
+				PredictionOptions: schema.PredictionOptions{
+					BasicModelRequest: schema.BasicModelRequest{Model: "bad.gguf"},
+				},
+			}
+			cfg.SetDefaults(ModelPath(dir))
+
+			Expect(cfg.ContextSize).NotTo(BeNil())
+		})
+	})
+
 	Context("PromptCacheAll default", func() {
 		It("defaults to true when omitted from YAML", func() {
 			cfg := &ModelConfig{}

core/config/meta/build.go

@@ -93,6 +93,9 @@ func applyOverride(f *FieldMeta, o FieldMetaOverride) {
 	if o.Component != "" {
 		f.Component = o.Component
 	}
+	if o.Language != "" {
+		f.Language = o.Language
+	}
 	if o.Placeholder != "" {
 		f.Placeholder = o.Placeholder
 	}

core/config/meta/constants.go

@@ -8,6 +8,7 @@ const (
 	ProviderModelsTTS        = "models:tts"
 	ProviderModelsTranscript = "models:transcript"
 	ProviderModelsVAD        = "models:vad"
+	ProviderModelsScore      = "models:score"
 )
 
 // Static option lists embedded directly in field metadata.

core/config/meta/registry.go

@@ -226,6 +226,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Label:       "Chat Template",
 			Description: "Go template for chat completion requests",
 			Component:   "code-editor",
+			Language:    "gotemplate",
 			Order:       40,
 		},
 		"template.chat_message": {
@@ -233,6 +234,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Label:       "Chat Message Template",
 			Description: "Go template for individual chat messages",
 			Component:   "code-editor",
+			Language:    "gotemplate",
 			Order:       41,
 		},
 		"template.completion": {
@@ -240,13 +242,22 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Label:       "Completion Template",
 			Description: "Go template for completion requests",
 			Component:   "code-editor",
+			Language:    "gotemplate",
 			Order:       42,
 		},
+		"template.function": {
+			Section:     "templates",
+			Label:       "Functions Template",
+			Description: "Go template applied when tools/functions are present in the request",
+			Component:   "code-editor",
+			Language:    "gotemplate",
+			Order:       43,
+		},
 		"template.use_tokenizer_template": {
 			Section:     "templates",
 			Label:       "Use Tokenizer Template",
 			Description: "Use the chat template from the model's tokenizer config",
-			Order:       43,
+			Order:       44,
 		},
 		// Router section template — kept in the templates UI section
 		// (rather than the router section under "other") so operators
@@ -257,7 +268,8 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Label:       "Router Classifier System Prompt",
 			Description: "Go text/template (with sprig functions) for the routing system prompt the score classifier feeds to its classifier_model. Executed with `.Policies` ([]{Label, Description}). Empty falls back to the built-in Arch-Router-shaped prompt (route-listing block + JSON output schema). Override when the classifier model was trained on a different schema or you need the routing instructions in a different language. The candidate format scored against the model is fixed at `{\"route\": \"<label>\"}` — keep your override's output schema instruction matching that.",
 			Component:   "code-editor",
-			Order:       44,
+			Language:    "gotemplate",
+			Order:       45,
 		},
 
 		// --- Pipeline ---
@@ -308,6 +320,41 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			},
 			Order: 64,
 		},
+		"pipeline.disable_thinking": {
+			Section:     "pipeline",
+			Label:       "Disable Thinking",
+			Description: "Suppress reasoning/thinking output from the pipeline LLM (sets enable_thinking=false on the underlying model). Use for models that emit <think> blocks you don't want spoken or streamed back to the realtime client.",
+			Component:   "toggle",
+			Order:       65,
+		},
+		"pipeline.streaming.llm": {
+			Section:     "pipeline",
+			Label:       "Stream LLM",
+			Description: "Stream LLM tokens to the realtime client as they are generated instead of waiting for the full response. Emits incremental response.output_audio_transcript.delta / text deltas.",
+			Component:   "toggle",
+			Order:       66,
+		},
+		"pipeline.streaming.tts": {
+			Section:     "pipeline",
+			Label:       "Stream TTS",
+			Description: "Stream synthesized audio chunks to the realtime client as they are produced (requires a TTS backend that implements TTSStream). Falls back to unary synthesis otherwise.",
+			Component:   "toggle",
+			Order:       67,
+		},
+		"pipeline.streaming.transcription": {
+			Section:     "pipeline",
+			Label:       "Stream Transcription",
+			Description: "Stream partial transcription text to the realtime client as the STT backend produces it (requires a transcription backend that implements AudioTranscriptionStream). Falls back to unary transcription otherwise.",
+			Component:   "toggle",
+			Order:       68,
+		},
+		"pipeline.streaming.clause_chunking": {
+			Section:     "pipeline",
+			Label:       "Clause Chunking",
+			Description: "Split the streamed reply into speakable clauses and synthesize each as soon as it completes, instead of buffering the whole message before TTS — lower time-to-first-audio. Script-aware (handles CJK 。！？ and Thai/Lao spaces), so it does not whitespace-split. Requires Stream LLM; off buffers the whole message.",
+			Component:   "toggle",
+			Order:       69,
+		},
 
 		// --- Functions ---
 		"function.grammar.parallel_calls": {
@@ -365,14 +412,14 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 
 		// --- PII filtering (per-model) ---
 		"pii.enabled": {
-			Section:     "other",
+			Section:     "pii",
 			Label:       "PII Filtering Enabled",
 			Description: "Enable PII redaction middleware for this model. Unset means use the default (off for local backends, on for proxy-* / cloud-hosted backends).",
 			Component:   "toggle",
 			Order:       200,
 		},
 		"pii.patterns": {
-			Section:     "other",
+			Section:     "pii",
 			Label:       "PII Pattern Overrides",
 			Description: "Override the global default action for specific patterns on this model. Patterns not listed here inherit the global action (Settings → Middleware → Filtering).",
 			Component:   "pii-pattern-list",
@@ -385,7 +432,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 		// fails closed — the chat handler does NOT silently fall back
 		// to the local gRPC pipeline.
 		"proxy.mode": {
-			Section:     "other",
+			Section:     "proxy",
 			Label:       "Proxy Mode",
 			Description: "passthrough forwards the client's OpenAI body verbatim — point upstream_url at an OpenAI-compatible endpoint (incl. Anthropic's /v1/chat/completions compat layer). translate converts OpenAI ↔ Anthropic Messages so you can target a native API (/v1/messages); tool_calls and usage tokens survive the round-trip.",
 			Component:   "select",
@@ -397,7 +444,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order:   208,
 		},
 		"proxy.provider": {
-			Section:     "other",
+			Section:     "proxy",
 			Label:       "Proxy Provider",
 			Description: "Upstream API family. Drives auth header shape (Bearer vs x-api-key + anthropic-version) and, in translate mode, which request/response codec is used.",
 			Component:   "select",
@@ -409,28 +456,28 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order:   209,
 		},
 		"proxy.upstream_url": {
-			Section:     "other",
+			Section:     "proxy",
 			Label:       "Proxy Upstream URL",
 			Description: "Full POST endpoint of the upstream provider (e.g. https://api.openai.com/v1/chat/completions). Only used when Backend is cloud-proxy.",
 			Component:   "input",
 			Order:       210,
 		},
 		"proxy.api_key_env": {
-			Section:     "other",
+			Section:     "proxy",
 			Label:       "Proxy API Key Env Var",
 			Description: "Name of the environment variable holding the upstream API key. Reading from env keeps the secret out of the YAML and the admin UI.",
 			Component:   "input",
 			Order:       211,
 		},
 		"proxy.upstream_model": {
-			Section:     "other",
+			Section:     "proxy",
 			Label:       "Proxy Upstream Model",
 			Description: "Model name sent to the upstream. Leave empty to forward the client's model field unchanged. Useful when the LocalAI alias differs from the upstream's canonical name.",
 			Component:   "input",
 			Order:       212,
 		},
 		"proxy.request_timeout_seconds": {
-			Section:     "other",
+			Section:     "proxy",
 			Label:       "Proxy Request Timeout (seconds)",
 			Description: "Caps the upstream HTTP request duration. 0 disables the deadline; the request still ends when the client disconnects.",
 			Component:   "number",
@@ -445,7 +492,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 		// A host claimed by two configs is a critical error — the
 		// listener refuses to start until resolved.
 		"mitm.hosts": {
-			Section:     "other",
+			Section:     "mitm",
 			Label:       "MITM Intercept Hosts",
 			Description: "Hostnames the cloudproxy MITM proxy terminates TLS for on behalf of this model config. PII filtering and pattern overrides flow from this model when the host is intercepted. Each host must be unique across all configs.",
 			Component:   "string-list",
@@ -460,7 +507,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 		// the middleware admin page surfaces every model with a router
 		// block.
 		"router.classifier": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "Classifier",
 			Description: "Picks a candidate by scoring every policy label against the prompt. Only \"score\" is shipped today; it asks the classifier_model to rank each label and reads off the softmax. Empty defaults to \"score\".",
 			Component:   "select",
@@ -470,15 +517,15 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order: 230,
 		},
 		"router.classifier_model": {
-			Section:              "other",
+			Section:              "router",
 			Label:                "Classifier Model",
 			Description:          "Loaded LocalAI model the score classifier asks to rank each policy label as a continuation. Must support the Score gRPC primitive (today: llama-cpp, vLLM) and use the ChatML template. Arch-Router-1.5B Q4_K_M is the canonical choice; any small ChatML instruct model also works at a higher activation_threshold.",
 			Component:            "model-select",
-			AutocompleteProvider: ProviderModelsChat,
+			AutocompleteProvider: ProviderModelsScore,
 			Order:                231,
 		},
 		"router.fallback": {
-			Section:              "other",
+			Section:              "router",
 			Label:                "Fallback Model",
 			Description:          "Model used when no candidate's labels cover the classifier's active label set, or when the classifier errors. Empty means router failures bubble up as HTTP 500 — fail-fast, not silent-bypass.",
 			Component:            "model-select",
@@ -486,7 +533,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order:                232,
 		},
 		"router.activation_threshold": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "Activation Threshold",
 			Description: "Softmax-probability floor a policy must clear to join the active label set for a request. Higher → single-label dominant routes; lower → more multi-label activations. 0 picks the package default (0.15). On Arch-Router-1.5B a value around 0.40 keeps the dominant label clean without losing genuine compound activations.",
 			Component:   "slider",
@@ -496,7 +543,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order:       233,
 		},
 		"router.classifier_cache_size": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "Classifier L1 Cache Size",
 			Description: "Bounded LRU keyed on (case-folded, whitespace-trimmed) prompt — amortises the classifier round-trip across verbatim repeats common in agent loops. 0 here means \"use the default\" (1024); the cache cannot be disabled from YAML.",
 			Component:   "number",
@@ -504,21 +551,21 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order:       234,
 		},
 		"router.policies": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "Policies",
 			Description: "Label vocabulary the classifier scores over. Each policy has a label and a short natural-language description fed verbatim to the classifier model. Short action-oriented sentences work best (\"writing or debugging code\"; \"small talk\").",
 			Component:   "router-policies",
 			Order:       235,
 		},
 		"router.candidates": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "Candidates",
 			Description: "Routing table: each entry binds a downstream model to a set of policy labels it can serve. Order matters — the middleware picks the FIRST candidate whose labels are a superset of the active set, so list candidates smallest → largest.",
 			Component:   "router-candidates",
 			Order:       236,
 		},
 		"router.score_normalization": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "Score Normalization",
 			Description: "How the score classifier collapses per-candidate joint log-probs into the softmax input. \"raw\" (default) feeds joint log-prob as-is — on-distribution for Arch-Router (the route the model would actually emit if decoded freely). \"mean\" divides by candidate token count — fairer to long labels but off-distribution for models trained to emit fixed-format outputs.",
 			Component:   "select",
@@ -530,7 +577,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order: 240,
 		},
 		"router.embedding_cache.embedding_model": {
-			Section:              "other",
+			Section:              "router",
 			Label:                "L2 Cache: Embedding Model",
 			Description:          "Embedding model used by the L2 decision cache. Embeds incoming probes and looks them up in the per-router local-store collection. Empty disables the cache entirely. nomic-embed-text-v1.5 is the recommended default.",
 			Component:            "model-select",
@@ -538,7 +585,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order:                237,
 		},
 		"router.embedding_cache.similarity_threshold": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "L2 Cache: Similarity Threshold",
 			Description: "Cosine-similarity floor a cache candidate must clear to count as a hit. 0 picks the package default (0.80). Re-tune per embedding model — the histogram on the Routing tab shows where the cosine distribution actually sits.",
 			Component:   "slider",
@@ -548,7 +595,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order:       238,
 		},
 		"router.embedding_cache.confidence_threshold": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "L2 Cache: Confidence Threshold",
 			Description: "Minimum top-label probability a classifier decision must have to be inserted into the cache. 0 picks the package default (0.60). Uncertain decisions are skipped so they can't poison future paraphrases.",
 			Component:   "slider",
@@ -558,7 +605,7 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Order:       239,
 		},
 		"router.embedding_cache.store_name": {
-			Section:     "other",
+			Section:     "router",
 			Label:       "L2 Cache: Store Name",
 			Description: "Optional override for the local-store collection used by this router's cache. Empty defaults to \"router-cache-<router-model-name>\". Two routers sharing a store_name share their cache (rare).",
 			Component:   "input",

core/config/meta/registry_coverage_test.go

@@ -240,7 +240,6 @@ var grandfatheredUnregistered = []string{
 	"swap_space",
 	"system_prompt",
 	"template.edit",
-	"template.function",
 	"template.join_chat_messages_by_character",
 	"template.multimodal",
 	"template.reply_prefix",

core/config/meta/types.go

@@ -11,6 +11,7 @@ type FieldMeta struct {
 	Label       string        `json:"label"`                 // human-readable label
 	Description string        `json:"description,omitempty"` // help text
 	Component   string        `json:"component"`             // "input", "number", "toggle", "select", "slider", etc.
+	Language    string        `json:"language,omitempty"`    // syntax mode for code-editor fields: "yaml" (default), "gotemplate"
 	Placeholder string        `json:"placeholder,omitempty"`
 	Default     any           `json:"default,omitempty"`
 	Min         *float64      `json:"min,omitempty"`
@@ -51,6 +52,7 @@ type FieldMetaOverride struct {
 	Label                string
 	Description          string
 	Component            string
+	Language             string
 	Placeholder          string
 	Default              any
 	Min                  *float64
@@ -78,6 +80,10 @@ func DefaultSections() []Section {
 		{ID: "grpc", Label: "gRPC", Icon: "server", Order: 65},
 		{ID: "agent", Label: "Agent", Icon: "bot", Order: 70},
 		{ID: "mcp", Label: "MCP", Icon: "plug", Order: 75},
+		{ID: "router", Label: "Router", Icon: "git-merge", Order: 78},
+		{ID: "proxy", Label: "Proxy", Icon: "cloud", Order: 80},
+		{ID: "mitm", Label: "MITM Proxy", Icon: "shield", Order: 82},
+		{ID: "pii", Label: "PII", Icon: "shield", Order: 84},
 		{ID: "other", Label: "Other", Icon: "more-horizontal", Order: 100},
 	}
 }

core/config/model_config.go

@@ -385,7 +385,7 @@ type PIIConfig struct {
 	Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"`
 
 	// Patterns lets a model upgrade or downgrade individual pattern
-	// actions (mask | block | route_local) relative to the global
+	// actions (mask | block | allow) relative to the global
 	// defaults loaded from --pii-config / DefaultPatterns. Pattern IDs
 	// not listed inherit the global action. The regex itself stays
 	// global — only the action is settable per-model.
@@ -499,6 +499,16 @@ type Pipeline struct {
 	// the pipeline's LLM without editing the LLM model config. Overrides the LLM's
 	// own reasoning_effort. Unset leaves the LLM model config in charge.
 	ReasoningEffort string `yaml:"reasoning_effort,omitempty" json:"reasoning_effort,omitempty"`
+
+	// Streaming opts each pipeline stage into incremental delivery (LLM tokens,
+	// TTS audio chunks, transcription text). Unset stages keep the blocking
+	// unary path, so existing configs are unaffected.
+	Streaming PipelineStreaming `yaml:"streaming,omitempty" json:"streaming,omitempty"`
+
+	// DisableThinking suppresses reasoning/thinking for the pipeline LLM (maps
+	// to enable_thinking=false backend metadata) without editing the underlying
+	// LLM model config. Unset leaves the LLM model config in charge.
+	DisableThinking *bool `yaml:"disable_thinking,omitempty" json:"disable_thinking,omitempty"`
 }
 
 // ApplyReasoningEffort resolves the effective reasoning effort — a per-request
@@ -530,6 +540,41 @@ func (c *ModelConfig) ApplyReasoningEffort(requestEffort string) {
 	}
 }
 
+// @Description PipelineStreaming toggles incremental delivery per realtime stage.
+type PipelineStreaming struct {
+	LLM           *bool `yaml:"llm,omitempty" json:"llm,omitempty"`
+	TTS           *bool `yaml:"tts,omitempty" json:"tts,omitempty"`
+	Transcription *bool `yaml:"transcription,omitempty" json:"transcription,omitempty"`
+	// ClauseChunking splits the streamed LLM reply into speakable clauses and
+	// synthesizes each as soon as it completes, instead of buffering the whole
+	// message before TTS. Script-aware (CJK/Thai), so it does not rely on
+	// whitespace sentence boundaries. Requires LLM streaming; unset buffers the
+	// whole message (today's default).
+	ClauseChunking *bool `yaml:"clause_chunking,omitempty" json:"clause_chunking,omitempty"`
+}
+
+// StreamLLM reports whether LLM tokens should be streamed for this pipeline.
+func (p Pipeline) StreamLLM() bool { return p.Streaming.LLM != nil && *p.Streaming.LLM }
+
+// StreamTTS reports whether TTS audio should be streamed for this pipeline.
+func (p Pipeline) StreamTTS() bool { return p.Streaming.TTS != nil && *p.Streaming.TTS }
+
+// StreamTranscription reports whether transcription text should be streamed.
+func (p Pipeline) StreamTranscription() bool {
+	return p.Streaming.Transcription != nil && *p.Streaming.Transcription
+}
+
+// ChunkClauses reports whether the streamed reply should be split into
+// script-aware clauses and synthesized incrementally rather than buffered whole.
+func (p Pipeline) ChunkClauses() bool {
+	return p.Streaming.ClauseChunking != nil && *p.Streaming.ClauseChunking
+}
+
+// ThinkingDisabled reports whether the pipeline forces the LLM's thinking off.
+func (p Pipeline) ThinkingDisabled() bool {
+	return p.DisableThinking != nil && *p.DisableThinking
+}
+
 // @Description File configuration for model downloads
 type File struct {
 	Filename string         `yaml:"filename,omitempty" json:"filename,omitempty"`
@@ -1229,14 +1274,20 @@ func (c *ModelConfig) GuessUsecases(u ModelConfigUsecase) bool {
 	}
 
 	if (u & FLAG_CHAT) == FLAG_CHAT {
-		if c.TemplateConfig.Chat == "" && c.TemplateConfig.ChatMessage == "" && !c.TemplateConfig.UseTokenizerTemplate {
-			return false
-		}
-		if slices.Contains(nonTextGenBackends, c.Backend) {
-			return false
-		}
-		if c.Embeddings != nil && *c.Embeddings {
-			return false
+		// A router model is a chat dispatcher: it carries no chat
+		// template of its own (those live on the candidates it routes
+		// to) and is invoked through the chat endpoint, so the router
+		// block stands in for chat capability.
+		if !c.HasRouter() {
+			if c.TemplateConfig.Chat == "" && c.TemplateConfig.ChatMessage == "" && !c.TemplateConfig.UseTokenizerTemplate {
+				return false
+			}
+			if slices.Contains(nonTextGenBackends, c.Backend) {
+				return false
+			}
+			if c.Embeddings != nil && *c.Embeddings {
+				return false
+			}
 		}
 	}
 	if (u & FLAG_COMPLETION) == FLAG_COMPLETION {

core/config/model_config_test.go

@@ -283,6 +283,18 @@ parameters:
 		Expect(e.HasUsecases(FLAG_CHAT)).To(BeFalse())
 		Expect(e.HasUsecases(FLAG_EMBEDDINGS)).To(BeTrue())
 
+		// Router models are chat dispatchers: no chat template of their
+		// own, but invoked through the chat endpoint, so they default to
+		// chat-capable.
+		r := ModelConfig{
+			Name: "r",
+			Router: RouterConfig{
+				Candidates: []RouterCandidate{{Model: "downstream", Labels: []string{"general"}}},
+			},
+		}
+		Expect(r.HasUsecases(FLAG_ANY)).To(BeTrue())
+		Expect(r.HasUsecases(FLAG_CHAT)).To(BeTrue())
+
 		f := ModelConfig{
 			Name:    "f",
 			Backend: "piper",

core/config/mtp.go

@@ -30,11 +30,26 @@ func MTPSpecOptions() []string {
 	return out
 }
 
-// HasEmbeddedMTPHead reports whether the parsed GGUF declares a Multi-Token
-// Prediction head. Detection reads `<arch>.nextn_predict_layers`, which is
-// what `gguf_writer.add_nextn_predict_layers(n)` emits in upstream's
+// isDraftOnlyAssistantArch reports whether an architecture names a standalone
+// MTP *draft* model rather than a self-speculating trunk. Upstream's Gemma4 MTP
+// (ggml-org/llama.cpp#23398) registers the head as a separate `gemma4-assistant`
+// architecture whose GGUF still carries `nextn_predict_layers`, but which cannot
+// run alone: it requires a paired target context (`ctx_other`). Such archs must
+// not trigger the embedded-head self-speculation defaults. The `-assistant`
+// suffix is upstream's naming convention for these draft-only checkpoints.
+func isDraftOnlyAssistantArch(arch string) bool {
+	return strings.HasSuffix(arch, "-assistant")
+}
+
+// HasEmbeddedMTPHead reports whether the parsed GGUF declares a self-speculating
+// Multi-Token Prediction head. Detection reads `<arch>.nextn_predict_layers`,
+// which is what `gguf_writer.add_nextn_predict_layers(n)` emits in upstream's
 // `conversion/qwen.py` MTP mixin. A positive layer count means the head is
 // present in the same GGUF as the trunk.
+//
+// Draft-only assistant architectures (e.g. Gemma4's `gemma4-assistant`) carry
+// the same key but are separate draft checkpoints meant to be paired with a
+// target model, so they are deliberately excluded here.
 func HasEmbeddedMTPHead(f *gguf.GGUFFile) (uint32, bool) {
 	if f == nil {
 		return 0, false
@@ -43,6 +58,9 @@ func HasEmbeddedMTPHead(f *gguf.GGUFFile) (uint32, bool) {
 	if arch == "" {
 		return 0, false
 	}
+	if isDraftOnlyAssistantArch(arch) {
+		return 0, false
+	}
 	v, ok := f.Header.MetadataKV.Get(arch + ".nextn_predict_layers")
 	if !ok {
 		return 0, false

core/config/mtp_test.go

@@ -3,10 +3,33 @@ package config_test
 import (
 	. "github.com/mudler/LocalAI/core/config"
 
+	gguf "github.com/gpustack/gguf-parser-go"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
+// ggufWithArch fabricates a minimal in-memory GGUF carrying the given
+// `general.architecture` and a positive `<arch>.nextn_predict_layers` count,
+// so HasEmbeddedMTPHead can be exercised without a real model file.
+func ggufWithArch(arch string, nextn uint32) *gguf.GGUFFile {
+	return &gguf.GGUFFile{
+		Header: gguf.GGUFHeader{
+			MetadataKV: gguf.GGUFMetadataKVs{
+				{
+					Key:       "general.architecture",
+					ValueType: gguf.GGUFMetadataValueTypeString,
+					Value:     arch,
+				},
+				{
+					Key:       arch + ".nextn_predict_layers",
+					ValueType: gguf.GGUFMetadataValueTypeUint32,
+					Value:     nextn,
+				},
+			},
+		},
+	}
+}
+
 var _ = Describe("MTP auto-defaults", func() {
 	Context("MTPSpecOptions", func() {
 		It("returns the upstream-recommended speculative tuple", func() {
@@ -82,5 +105,20 @@ var _ = Describe("MTP auto-defaults", func() {
 			Expect(ok).To(BeFalse())
 			Expect(n).To(BeZero())
 		})
+
+		It("detects a same-GGUF embedded head (DeepSeek/Qwen style)", func() {
+			n, ok := HasEmbeddedMTPHead(ggufWithArch("qwen3moe", 1))
+			Expect(ok).To(BeTrue())
+			Expect(n).To(Equal(uint32(1)))
+		})
+
+		It("ignores a gemma4-assistant draft-only model", func() {
+			// The assistant GGUF carries nextn_predict_layers but is a separate
+			// draft model that requires a paired target (ctx_other); it cannot
+			// self-speculate, so it must not trigger the embedded-head defaults.
+			n, ok := HasEmbeddedMTPHead(ggufWithArch("gemma4-assistant", 48))
+			Expect(ok).To(BeFalse())
+			Expect(n).To(BeZero())
+		})
 	})
 })

core/config/pipeline_streaming_test.go

@@ -0,0 +1,57 @@
+package config
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+)
+
+// The realtime pipeline can stream each stage (LLM tokens, TTS audio,
+// transcription text) and can disable model "thinking" for the LLM. These are
+// opt-in per pipeline; everything defaults to off so existing configs keep the
+// unary behaviour.
+var _ = Describe("Pipeline streaming config", func() {
+	It("defaults every streaming + thinking helper to false when unset", func() {
+		var p Pipeline
+		Expect(p.StreamLLM()).To(BeFalse())
+		Expect(p.StreamTTS()).To(BeFalse())
+		Expect(p.StreamTranscription()).To(BeFalse())
+		Expect(p.ChunkClauses()).To(BeFalse())
+		Expect(p.ThinkingDisabled()).To(BeFalse())
+	})
+
+	It("parses the nested streaming block and disable_thinking from YAML", func() {
+		var c ModelConfig
+		err := yaml.Unmarshal([]byte(`
+name: gpt-realtime
+pipeline:
+  llm: my-llm
+  tts: my-tts
+  transcription: my-stt
+  streaming:
+    llm: true
+    tts: true
+    transcription: true
+    clause_chunking: true
+  disable_thinking: true
+`), &c)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(c.Pipeline.StreamLLM()).To(BeTrue())
+		Expect(c.Pipeline.StreamTTS()).To(BeTrue())
+		Expect(c.Pipeline.StreamTranscription()).To(BeTrue())
+		Expect(c.Pipeline.ChunkClauses()).To(BeTrue())
+		Expect(c.Pipeline.ThinkingDisabled()).To(BeTrue())
+	})
+
+	It("treats an explicit false in the streaming block as disabled", func() {
+		var c ModelConfig
+		err := yaml.Unmarshal([]byte(`
+name: gpt-realtime
+pipeline:
+  streaming:
+    tts: false
+`), &c)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(c.Pipeline.StreamTTS()).To(BeFalse())
+	})
+})

core/gallery/backends_test.go

@@ -50,7 +50,14 @@ var _ = Describe("Runtime capability-based backend selection", func() {
 		must(os.WriteFile(filepath.Join(cudaDir, "metadata.json"), b, 0o644))
 		must(os.WriteFile(filepath.Join(cudaDir, "run.sh"), []byte(""), 0o755))
 
-		// Default system: alias should point to CPU
+		// Default system: alias should point to CPU. Force the capability to
+		// "cpu" so this is hermetic on hosts that actually have a GPU: backend
+		// preference keys off getSystemCapabilities() (env → real nvidia-smi
+		// detection), not GPUVendor, so without this a GPU dev box reports
+		// "nvidia" and the cuda alias wins. The NVIDIA case below overrides it.
+		must(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "cpu"))
+		defer func() { _ = os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") }()
+
 		sysDefault, err := system.GetSystemState(
 			system.WithBackendPath(tempDir),
 		)

core/gallery/importers/importers.go

@@ -158,6 +158,11 @@ var defaultImporters = []Importer{
 	// RFDetrImporter must run before TransformersImporter — RF-DETR
 	// checkpoints may carry tokenizer-adjacent artefacts.
 	&RFDetrImporter{},
+	// LocateAnythingImporter (NVIDIA LocateAnything open-vocab detection,
+	// native C++/ggml port) must run before LlamaCPPImporter so its GGUF
+	// bundles aren't claimed by the generic .gguf importer; kept next to
+	// RFDetrImporter as both are detection models.
+	&LocateAnythingImporter{},
 	// Existing
 	// DS4Importer must precede LlamaCPPImporter - ds4 weights are GGUFs and
 	// would otherwise be claimed by the generic .gguf-handling llama-cpp

core/gallery/importers/locate-anything.go

@@ -0,0 +1,137 @@
+package importers
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/core/schema"
+	"go.yaml.in/yaml/v2"
+)
+
+var _ Importer = &LocateAnythingImporter{}
+
+// LocateAnythingImporter routes NVIDIA LocateAnything open-vocabulary
+// object-detection / visual-grounding repositories to the
+// "locate-anything-cpp" backend (a native C++/ggml port). It must be
+// registered BEFORE the generic GGUF matchers (LlamaCPPImporter) so its
+// GGUF bundles aren't swallowed by the generic .gguf-handling importer,
+// and alongside RFDetrImporter since both are detection models that may
+// carry tokenizer-adjacent artefacts.
+//
+// Detection signals:
+//   - preferences.backend="locate-anything-cpp" (explicit override);
+//   - repo name contains "locate-anything" or "locateanything"
+//     (case-insensitive).
+type LocateAnythingImporter struct{}
+
+func (i *LocateAnythingImporter) Name() string      { return "locate-anything-cpp" }
+func (i *LocateAnythingImporter) Modality() string  { return "detection" }
+func (i *LocateAnythingImporter) AutoDetects() bool { return true }
+
+func repoLooksLikeLocateAnything(repo string) bool {
+	lower := strings.ToLower(repo)
+	return strings.Contains(lower, "locate-anything") ||
+		strings.Contains(lower, "locateanything") ||
+		strings.Contains(lower, "locate-anything.cpp") ||
+		strings.Contains(lower, "locate-anything-cpp")
+}
+
+func (i *LocateAnythingImporter) Match(details Details) bool {
+	preferences, err := details.Preferences.MarshalJSON()
+	if err != nil {
+		return false
+	}
+	preferencesMap := make(map[string]any)
+	if len(preferences) > 0 {
+		if err := json.Unmarshal(preferences, &preferencesMap); err != nil {
+			return false
+		}
+	}
+
+	if b, ok := preferencesMap["backend"].(string); ok && b == "locate-anything-cpp" {
+		return true
+	}
+
+	if details.HuggingFace != nil {
+		repoName := details.HuggingFace.ModelID
+		if idx := strings.Index(repoName, "/"); idx >= 0 {
+			repoName = repoName[idx+1:]
+		}
+		if repoLooksLikeLocateAnything(repoName) {
+			return true
+		}
+	}
+
+	// Fallback: hfapi recursion bug may leave HuggingFace nil — decide
+	// from the URI owner/repo.
+	if _, repo, ok := HFOwnerRepoFromURI(details.URI); ok {
+		if repoLooksLikeLocateAnything(repo) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (i *LocateAnythingImporter) Import(details Details) (gallery.ModelConfig, error) {
+	preferences, err := details.Preferences.MarshalJSON()
+	if err != nil {
+		return gallery.ModelConfig{}, err
+	}
+	preferencesMap := make(map[string]any)
+	if len(preferences) > 0 {
+		if err := json.Unmarshal(preferences, &preferencesMap); err != nil {
+			return gallery.ModelConfig{}, err
+		}
+	}
+
+	name, ok := preferencesMap["name"].(string)
+	if !ok {
+		name = filepath.Base(details.URI)
+	}
+
+	description, ok := preferencesMap["description"].(string)
+	if !ok {
+		description = "Imported from " + details.URI
+	}
+
+	// Prefer the canonical HF "owner/repo" identifier so the emitted
+	// YAML mirrors gallery locate-anything entries.
+	model := details.URI
+	if details.HuggingFace != nil && details.HuggingFace.ModelID != "" {
+		model = details.HuggingFace.ModelID
+	} else if owner, repo, ok := HFOwnerRepoFromURI(details.URI); ok {
+		model = owner + "/" + repo
+	}
+
+	// Always the native C++/ggml backend; explicit preferences.backend
+	// overrides the default.
+	backend := "locate-anything-cpp"
+	if b, ok := preferencesMap["backend"].(string); ok && b != "" {
+		backend = b
+	}
+
+	modelConfig := config.ModelConfig{
+		Name:                name,
+		Description:         description,
+		Backend:             backend,
+		KnownUsecaseStrings: []string{"detection"},
+		PredictionOptions: schema.PredictionOptions{
+			BasicModelRequest: schema.BasicModelRequest{Model: model},
+		},
+	}
+
+	data, err := yaml.Marshal(modelConfig)
+	if err != nil {
+		return gallery.ModelConfig{}, err
+	}
+
+	return gallery.ModelConfig{
+		Name:        name,
+		Description: description,
+		ConfigFile:  string(data),
+	}, nil
+}

core/gallery/importers/locate-anything_test.go

@@ -0,0 +1,218 @@
+package importers_test
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/mudler/LocalAI/core/gallery/importers"
+	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("LocateAnythingImporter", func() {
+	Context("Importer interface metadata", func() {
+		It("exposes name/modality/autodetect", func() {
+			imp := &importers.LocateAnythingImporter{}
+			Expect(imp.Name()).To(Equal("locate-anything-cpp"))
+			Expect(imp.Modality()).To(Equal("detection"))
+			Expect(imp.AutoDetects()).To(BeTrue())
+		})
+	})
+
+	Context("Match", func() {
+		It("matches when backend preference is locate-anything-cpp", func() {
+			imp := &importers.LocateAnythingImporter{}
+			preferences := json.RawMessage(`{"backend": "locate-anything-cpp"}`)
+			details := importers.Details{
+				URI:         "https://example.com/some-model",
+				Preferences: preferences,
+			}
+
+			Expect(imp.Match(details)).To(BeTrue())
+		})
+
+		It("matches when the repo name contains 'locate-anything' (case-insensitive)", func() {
+			imp := &importers.LocateAnythingImporter{}
+			details := importers.Details{
+				URI: "https://huggingface.co/mudler/locate-anything-cpp-3b",
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID: "mudler/Locate-Anything-CPP-3B",
+					Author:  "mudler",
+				},
+			}
+
+			Expect(imp.Match(details)).To(BeTrue())
+		})
+
+		It("matches when the repo name contains 'locateanything' (case-insensitive)", func() {
+			imp := &importers.LocateAnythingImporter{}
+			details := importers.Details{
+				URI: "https://huggingface.co/nvidia/LocateAnything-3B",
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID: "nvidia/LocateAnything-3B",
+					Author:  "nvidia",
+				},
+			}
+
+			Expect(imp.Match(details)).To(BeTrue())
+		})
+
+		It("matches via URI fallback when HuggingFace details are missing", func() {
+			imp := &importers.LocateAnythingImporter{}
+			details := importers.Details{
+				URI: "https://huggingface.co/nvidia/LocateAnything-3B",
+			}
+
+			Expect(imp.Match(details)).To(BeTrue())
+		})
+
+		It("does not match unrelated repos without locate-anything signals", func() {
+			imp := &importers.LocateAnythingImporter{}
+			details := importers.Details{
+				URI: "https://huggingface.co/meta-llama/Llama-3-8B",
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID: "meta-llama/Llama-3-8B",
+					Author:  "meta-llama",
+				},
+			}
+
+			Expect(imp.Match(details)).To(BeFalse())
+		})
+
+		It("does not match an rfdetr repo", func() {
+			imp := &importers.LocateAnythingImporter{}
+			details := importers.Details{
+				URI: "https://huggingface.co/mudler/rfdetr-cpp-nano",
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID: "mudler/rfdetr-cpp-nano",
+					Author:  "mudler",
+				},
+			}
+
+			Expect(imp.Match(details)).To(BeFalse())
+		})
+
+		It("returns false for invalid preferences JSON", func() {
+			imp := &importers.LocateAnythingImporter{}
+			preferences := json.RawMessage(`not valid json`)
+			details := importers.Details{
+				URI:         "https://example.com/model",
+				Preferences: preferences,
+			}
+
+			Expect(imp.Match(details)).To(BeFalse())
+		})
+	})
+
+	Context("Import", func() {
+		It("produces a YAML with backend locate-anything-cpp and the repo as the model", func() {
+			imp := &importers.LocateAnythingImporter{}
+			details := importers.Details{
+				URI: "https://huggingface.co/nvidia/LocateAnything-3B",
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID: "nvidia/LocateAnything-3B",
+					Author:  "nvidia",
+				},
+			}
+
+			modelConfig, err := imp.Import(details)
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: locate-anything-cpp"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("nvidia/LocateAnything-3B"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("detection"), fmt.Sprintf("Model config: %+v", modelConfig))
+		})
+
+		It("respects custom name and description from preferences", func() {
+			imp := &importers.LocateAnythingImporter{}
+			preferences := json.RawMessage(`{"name": "my-locate", "description": "Custom"}`)
+			details := importers.Details{
+				URI:         "https://huggingface.co/nvidia/LocateAnything-3B",
+				Preferences: preferences,
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID: "nvidia/LocateAnything-3B",
+					Author:  "nvidia",
+				},
+			}
+
+			modelConfig, err := imp.Import(details)
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(modelConfig.Name).To(Equal("my-locate"))
+			Expect(modelConfig.Description).To(Equal("Custom"))
+		})
+	})
+
+	// Table-driven coverage of the backend routing: locate-anything repos
+	// always route to the native locate-anything-cpp backend, with an
+	// explicit preferences.backend override honoured.
+	//
+	// Cases are kept offline-deterministic by injecting Details directly
+	// rather than going through DiscoverModelConfig (which would hit live HF).
+	Context("backend routing (offline)", func() {
+		hfFile := func(path string) hfapi.ModelFile {
+			return hfapi.ModelFile{Path: path}
+		}
+
+		type tc struct {
+			name          string
+			uri           string
+			modelID       string
+			files         []hfapi.ModelFile
+			prefs         string
+			expectBackend string // expected `backend:` line content
+		}
+
+		entries := []tc{
+			{
+				name:          "canonical NVIDIA repo routes to locate-anything-cpp",
+				uri:           "https://huggingface.co/nvidia/LocateAnything-3B",
+				modelID:       "nvidia/LocateAnything-3B",
+				files:         []hfapi.ModelFile{hfFile("locate-anything-3b-q8_0.gguf"), hfFile("README.md")},
+				prefs:         "",
+				expectBackend: "backend: locate-anything-cpp",
+			},
+			{
+				name:          "GGUF bundle with locate-anything name routes to locate-anything-cpp",
+				uri:           "https://huggingface.co/mudler/locate-anything.cpp-3b",
+				modelID:       "mudler/locate-anything.cpp-3b",
+				files:         []hfapi.ModelFile{hfFile("model-f16.gguf")},
+				prefs:         "",
+				expectBackend: "backend: locate-anything-cpp",
+			},
+			{
+				name:          "explicit preferences.backend override is honoured",
+				uri:           "https://huggingface.co/nvidia/LocateAnything-3B",
+				modelID:       "nvidia/LocateAnything-3B",
+				files:         nil,
+				prefs:         `{"backend": "locate-anything-cpp"}`,
+				expectBackend: "backend: locate-anything-cpp",
+			},
+		}
+
+		for _, e := range entries {
+			e := e // capture for closure
+			It(e.name, func() {
+				imp := &importers.LocateAnythingImporter{}
+				details := importers.Details{
+					URI: e.uri,
+					HuggingFace: &hfapi.ModelDetails{
+						ModelID: e.modelID,
+						Files:   e.files,
+					},
+				}
+				if e.prefs != "" {
+					details.Preferences = json.RawMessage(e.prefs)
+				}
+
+				Expect(imp.Match(details)).To(BeTrue(), fmt.Sprintf("Match should fire for %+v", details))
+
+				modelConfig, err := imp.Import(details)
+				Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Import error: %v", err))
+				Expect(modelConfig.ConfigFile).To(ContainSubstring(e.expectBackend),
+					fmt.Sprintf("Model config: %+v", modelConfig))
+			})
+		}
+	})
+})

core/gallery/importers/mlx.go

@@ -64,7 +64,17 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) {
 		description = "Imported from " + details.URI
 	}
 
+	// Vision-language checkpoints (e.g. gemma-4 E4B) declare the
+	// "image-text-to-text" pipeline tag on HuggingFace. The text-only mlx-lm
+	// tokenizer does not carry their processor chat template, so routing them
+	// through the plain mlx backend yields degenerate looping output
+	// (issue #10269). Send them to the mlx-vlm backend, which applies the
+	// processor-aware chat template.
 	backend := "mlx"
+	if details.HuggingFace != nil && details.HuggingFace.PipelineTag == "image-text-to-text" {
+		backend = "mlx-vlm"
+	}
+	// An explicit backend preference always wins.
 	b, ok := preferencesMap["backend"].(string)
 	if ok {
 		backend = b

core/gallery/importers/mlx_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 
 	"github.com/mudler/LocalAI/core/gallery/importers"
+	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -122,6 +123,60 @@ var _ = Describe("MLXImporter", func() {
 			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm"))
 		})
 
+		It("should auto-route vision-language models to the mlx-vlm backend", func() {
+			// gemma-4 E4B and similar VLMs declare pipeline_tag
+			// "image-text-to-text" on HuggingFace. The text-only mlx-lm
+			// tokenizer does not carry their processor chat template, so
+			// routing them through the plain mlx backend produces degenerate
+			// looping output (issue #10269). They must go to mlx-vlm.
+			details := importers.Details{
+				URI: "https://huggingface.co/mlx-community/gemma-4-E4B-it-qat-4bit",
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID:     "mlx-community/gemma-4-E4B-it-qat-4bit",
+					PipelineTag: "image-text-to-text",
+				},
+			}
+
+			modelConfig, err := importer.Import(details)
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm"))
+		})
+
+		It("should keep text-only models on the plain mlx backend", func() {
+			details := importers.Details{
+				URI: "https://huggingface.co/mlx-community/Llama-3.2-1B-Instruct-4bit",
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID:     "mlx-community/Llama-3.2-1B-Instruct-4bit",
+					PipelineTag: "text-generation",
+				},
+			}
+
+			modelConfig, err := importer.Import(details)
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx"))
+			Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("backend: mlx-vlm"))
+		})
+
+		It("should honor an explicit backend preference even for a VLM", func() {
+			preferences := json.RawMessage(`{"backend": "mlx"}`)
+			details := importers.Details{
+				URI:         "https://huggingface.co/mlx-community/gemma-4-E4B-it-qat-4bit",
+				Preferences: preferences,
+				HuggingFace: &hfapi.ModelDetails{
+					ModelID:     "mlx-community/gemma-4-E4B-it-qat-4bit",
+					PipelineTag: "image-text-to-text",
+				},
+			}
+
+			modelConfig, err := importer.Import(details)
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx"))
+			Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("backend: mlx-vlm"))
+		})
+
 		It("should handle invalid JSON preferences", func() {
 			preferences := json.RawMessage(`invalid json`)
 			details := importers.Details{

core/http/endpoints/anthropic/messages.go

@@ -353,7 +353,7 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
 			overrides = make(map[string]pii.Action, len(raw))
 			for ovid, action := range raw {
 				switch pii.Action(action) {
-				case pii.ActionMask, pii.ActionBlock, pii.ActionRouteLocal:
+				case pii.ActionMask, pii.ActionBlock, pii.ActionAllow:
 					overrides[ovid] = pii.Action(action)
 				}
 			}

core/http/endpoints/localai/api_instructions.go

@@ -102,7 +102,7 @@ var instructionDefs = []instructionDef{
 		Name:        "pii-filtering",
 		Description: "Inspect and tune the regex PII filter applied to chat requests",
 		Tags:        []string{"pii"},
-		Intro:       "GET /api/pii/patterns lists the active pattern set with each one's action (mask, block, route_local). GET /api/pii/events returns recent redaction events filtered by correlation_id / user_id / pattern_id (admin or local-user only). POST /api/pii/test dry-runs the redactor against an admin-supplied string. POST /api/pii/decide is the programmatic decision oracle for external routers: send `{text}`, receive `{findings, suggested_action, redacted_preview}` without LocalAI mutating, recording, or acting on the call — caller composes the action with its own policy. Default patterns: email, phone, SSN, credit card (Luhn), IPv4, common API key prefixes (sk-, pk-, ghp_, github_pat_). PII is per-model: by default it is OFF for non-proxy backends and ON for backends starting with proxy-* (cloud passthroughs). Opt in with `pii: { enabled: true }` in a model's YAML; use `pii: { patterns: [{id, action}] }` to upgrade or downgrade individual actions for that model. Override global default actions via --pii-config pii.yaml; --disable-pii turns the filter off entirely.",
+		Intro:       "GET /api/pii/patterns lists the active pattern set with each one's action (mask, block, allow). GET /api/pii/events returns recent redaction events filtered by correlation_id / user_id / pattern_id (admin or local-user only). POST /api/pii/test dry-runs the redactor against an admin-supplied string. POST /api/pii/decide is the programmatic decision oracle for external routers: send `{text}`, receive `{findings, suggested_action, redacted_preview}` without LocalAI mutating, recording, or acting on the call — caller composes the action with its own policy. Default patterns: email, phone, SSN, credit card (Luhn), IPv4, common API key prefixes (sk-, pk-, ghp_, github_pat_). PII is per-model: by default it is OFF for non-proxy backends and ON for backends starting with proxy-* (cloud passthroughs). Opt in with `pii: { enabled: true }` in a model's YAML; use `pii: { patterns: [{id, action}] }` to upgrade or downgrade individual actions for that model. Override global default actions via --pii-config pii.yaml; --disable-pii turns the filter off entirely.",
 	},
 	{
 		Name:        "middleware-admin",

core/http/endpoints/localai/config_meta.go

@@ -124,6 +124,8 @@ func AutocompleteEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, a
 				filterFn = config.BuildUsecaseFilterFn(config.FLAG_VAD)
 			case config.UsecaseTranscript:
 				filterFn = config.BuildUsecaseFilterFn(config.FLAG_TRANSCRIPT)
+			case "score": // router classifier usecase (FLAG_SCORE); not in UsecaseInfoMap
+				filterFn = config.BuildUsecaseFilterFn(config.FLAG_SCORE)
 			default:
 				filterFn = config.NoFilterFn
 			}

core/http/endpoints/localai/pii_decide.go

@@ -15,9 +15,9 @@ import (
 //
 // External routers (e.g. the localai-org/platform router) call this
 // before dispatching to learn whether to mask the prompt in place,
-// route to a local-only backend, block the request, or pass it
-// through. LocalAI's in-band PII middleware is the alternative path
-// for direct-to-LocalAI clients — same Redactor, different framing.
+// block the request, or pass it through. LocalAI's in-band PII
+// middleware is the alternative path for direct-to-LocalAI clients —
+// same Redactor, different framing.
 //
 // Takes the *pii.Redactor directly rather than the whole
 // *application.Application so the handler stays unit-testable with a
@@ -62,24 +62,18 @@ func PIIDecideEndpoint(redactor *pii.Redactor) echo.HandlerFunc {
 	}
 }
 
-// actionAllow is the wire-only value for "no findings". The other
-// three map to existing pii.Action* constants; allow has no in-band
-// counterpart because the in-band middleware simply passes through.
-const actionAllow = "allow"
-
 // suggestedAction collapses the Redactor's Result flags onto a single
-// wire-format action using the in-band ordering (block > route_local
-// > mask > allow). Spans-without-Blocked-or-LocalOnly means every
-// match resolved to ActionMask.
+// wire-format action using the in-band ordering (block > mask >
+// allow). "allow" covers both "nothing matched" and "matched but every
+// span resolved to the allow action" — in both cases the caller may
+// dispatch unchanged, with the Findings list reporting what was seen.
 func suggestedAction(res pii.Result) string {
 	switch {
 	case res.Blocked:
 		return string(pii.ActionBlock)
-	case res.LocalOnly:
-		return string(pii.ActionRouteLocal)
-	case len(res.Spans) > 0:
+	case res.Masked:
 		return string(pii.ActionMask)
 	default:
-		return actionAllow
+		return string(pii.ActionAllow)
 	}
 }