From b35fdf0142c4a9ddfb10da2ea8854f17239b29e2 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Fri, 4 Sep 2026 08:49:55 +0200 Subject: [PATCH 01/85] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `fe215a8ccdce6b844d2a3a3bbde08ae76a6284bf` (#11864) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index ca4c45114..1d0304874 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=caf7eae5282d840d77e9f91a56df7d2ef28fa612 +IK_LLAMA_VERSION?=fe215a8ccdce6b844d2a3a3bbde08ae76a6284bf LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From 8b31f5d6ad93746f49144cfda3652af9aa6f84aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 08:51:51 +0200 Subject: [PATCH 02/85] chore(deps): bump grpcio from 1.83.0 to 1.83.1 in /backend/python/coqui (#11856) Bumps [grpcio](https://github.com/grpc/grpc) from 1.83.0 to 1.83.1. - [Release notes](https://github.com/grpc/grpc/releases) - [Commits](https://github.com/grpc/grpc/compare/v1.83.0...v1.83.1) --- updated-dependencies: - dependency-name: grpcio dependency-version: 1.83.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- backend/python/coqui/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/python/coqui/requirements.txt b/backend/python/coqui/requirements.txt index db72b4b98..305f95f12 100644 --- a/backend/python/coqui/requirements.txt +++ b/backend/python/coqui/requirements.txt @@ -1,4 +1,4 @@ -grpcio==1.83.0 +grpcio==1.83.1 protobuf certifi packaging==26.3 \ No newline at end of file From 072e367df8e8f528c7d073217f32cd78b6b931f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:00:00 +0200 Subject: [PATCH 03/85] chore(deps): bump grpcio from 1.82.1 to 1.83.1 in /backend/python/rerankers (#11852) chore(deps): bump grpcio in /backend/python/rerankers Bumps [grpcio](https://github.com/grpc/grpc) from 1.82.1 to 1.83.1. - [Release notes](https://github.com/grpc/grpc/releases) - [Commits](https://github.com/grpc/grpc/compare/v1.82.1...v1.83.1) --- updated-dependencies: - dependency-name: grpcio dependency-version: 1.83.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- backend/python/rerankers/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/python/rerankers/requirements.txt b/backend/python/rerankers/requirements.txt index 49943543f..5f1e5f0ae 100644 --- a/backend/python/rerankers/requirements.txt +++ b/backend/python/rerankers/requirements.txt @@ -1,3 +1,3 @@ -grpcio==1.82.1 +grpcio==1.83.1 protobuf certifi \ No newline at end of file From 5485a2e62bd0138d033e96c0faa77a550833f7ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:00:22 +0200 Subject: [PATCH 04/85] chore(deps): bump grpcio from 1.83.0 to 1.83.1 in /backend/python/vllm (#11853) Bumps [grpcio](https://github.com/grpc/grpc) from 1.83.0 to 1.83.1. - [Release notes](https://github.com/grpc/grpc/releases) - [Commits](https://github.com/grpc/grpc/compare/v1.83.0...v1.83.1) --- updated-dependencies: - dependency-name: grpcio dependency-version: 1.83.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- backend/python/vllm/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/python/vllm/requirements.txt b/backend/python/vllm/requirements.txt index 889f21d0d..ba7147632 100644 --- a/backend/python/vllm/requirements.txt +++ b/backend/python/vllm/requirements.txt @@ -1,4 +1,4 @@ -grpcio==1.83.0 +grpcio==1.83.1 protobuf certifi setuptools From 5e6a83694da4ff808734a477218aa5645367c6ad Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Fri, 4 Sep 2026 09:00:41 +0200 Subject: [PATCH 05/85] chore(model-gallery): :arrow_up: update checksum (#11868) :arrow_up: Checksum updates in gallery/index.yaml Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 4a0e6afa2..f716e60b7 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -353,7 +353,7 @@ files: - filename: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q4_K_M.gguf uri: huggingface://ibm-granite/granite-4.2-3b-GGUF/granite-4.2-3b-Q4_K_M.gguf - sha256: 20e436143017578687f7f848225cc6c6038126c84149192229c7dff6e4e0f427 + sha256: e0406663965846ae22a403456eb826ccce5f450840491f71952f18a7cb78e7d5 - !!merge <<: *granite-4-2-3b name: "granite-4.2-3b-q8" variants: [] @@ -378,7 +378,7 @@ files: - filename: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q8_0.gguf uri: huggingface://ibm-granite/granite-4.2-3b-GGUF/granite-4.2-3b-Q8_0.gguf - sha256: 9e97320b131445ab8d9098cafb48001e9925d879e71486a8af4db4c803c55394 + sha256: fbe986738041418e26de9e123ba740cb654931f85bf572a71bd01f9e6b85e53d - &granite-4-2-8b name: "granite-4.2-8b-q4" variants: @@ -4115,7 +4115,7 @@ uri: https://huggingface.co/LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V6-GGUF/resolve/main/Hermes3.6-35B-A3B-Uncensored-Genesis-V6-Q8_0.gguf - filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V6-Q8_0/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf uri: https://huggingface.co/LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V6-GGUF/resolve/main/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf - sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4 + sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc - &qwen3-6-35b-a3b-genesis-hermes-v7 name: "qwen3.6-35b-a3b-genesis-hermes-v7" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" @@ -4177,7 +4177,7 @@ uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-APEX.gguf - filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf - sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4 + sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc - !!merge <<: *qwen3-6-35b-a3b-genesis-hermes-v7 name: "qwen3.6-35b-a3b-genesis-hermes-v7-apex-compact" variants: [] @@ -4213,7 +4213,7 @@ uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-APEX-Compact.gguf - filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf - sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4 + sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc - !!merge <<: *qwen3-6-35b-a3b-genesis-hermes-v7 name: "qwen3.6-35b-a3b-genesis-hermes-v7-mtp-apex" variants: [] @@ -4261,7 +4261,7 @@ uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-MTP-APEX.gguf - filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf - sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4 + sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc - !!merge <<: *qwen3-6-35b-a3b-genesis-hermes-v7 name: "qwen3.6-35b-a3b-genesis-hermes-v7-mtp-apex-compact" variants: [] @@ -4309,7 +4309,7 @@ uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-MTP-APEX-Compact.gguf - filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf - sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4 + sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc - !!merge <<: *qwen3-6-35b-a3b-genesis-hermes-v7 name: "qwen3.6-35b-a3b-genesis-hermes-v7-q8-k-p" variants: [] @@ -4344,7 +4344,7 @@ uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-Q8_K_P.gguf - filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf - sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4 + sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc - &kat-coder-v2-5-dev name: "kat-coder-v2.5-dev" variants: From 1f2f931f532bea9ad34b05cc38165116f0bea2e2 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Fri, 4 Sep 2026 09:16:30 +0200 Subject: [PATCH 06/85] chore: :arrow_up: Update 0xShug0/audio.cpp to `8fe1ab805555c72aaf06d83828b523dfc6ed796d` (#11861) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 572667f60..dc4455c13 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=c18b7f737aac0a2855e9f963a427498739ad40fe +AUDIO_CPP_VERSION?=8fe1ab805555c72aaf06d83828b523dfc6ed796d AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From a742c87f76c4a35dafc2f68a70d56ce1b4d1b28b Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Fri, 4 Sep 2026 16:53:28 +0200 Subject: [PATCH 07/85] chore: :arrow_up: Update ServeurpersoCom/omnivoice.cpp to `78ce63c90d36b94a765073e8cbe0d241fc114590` (#11865) :arrow_up: Update ServeurpersoCom/omnivoice.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/omnivoice-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/omnivoice-cpp/Makefile b/backend/go/omnivoice-cpp/Makefile index 4cb089dab..26e90aa9e 100644 --- a/backend/go/omnivoice-cpp/Makefile +++ b/backend/go/omnivoice-cpp/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # omnivoice.cpp version OMNIVOICE_REPO?=https://github.com/ServeurpersoCom/omnivoice.cpp -OMNIVOICE_VERSION?=4f33af825d66e6ef1cb185e87b4589cacf747291 +OMNIVOICE_VERSION?=78ce63c90d36b94a765073e8cbe0d241fc114590 SO_TARGET?=libgomnivoicecpp.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From 87d15e7108f7a80ce68e2515d0abe512e036cd02 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Fri, 4 Sep 2026 16:57:26 +0200 Subject: [PATCH 08/85] chore: :arrow_up: Update NVIDIA/NeMo-Speech.cpp to `69d7fd46b253043ed6649bf35d68407448912a8e` (#11863) :arrow_up: Update NVIDIA/NeMo-Speech.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/nemo-speech-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/nemo-speech-cpp/Makefile b/backend/go/nemo-speech-cpp/Makefile index 24b944e11..0472e672c 100644 --- a/backend/go/nemo-speech-cpp/Makefile +++ b/backend/go/nemo-speech-cpp/Makefile @@ -12,7 +12,7 @@ # runs 'make -C backend/go/$(BACKEND) build' and then copies package/), so it # has to produce the binary and the package, not just the shared libraries. -NEMO_SPEECH_VERSION?=56b60d432f1731d6d5b28a4c5a31cbaf871daba1 +NEMO_SPEECH_VERSION?=69d7fd46b253043ed6649bf35d68407448912a8e NEMO_SPEECH_REPO?=https://github.com/NVIDIA/NeMo-Speech.cpp GOCMD?=go From 583992a76f3ed8235ce9213bfce276848755252d Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 5 Sep 2026 01:02:43 +0200 Subject: [PATCH 09/85] chore: :arrow_up: Update ggml-org/whisper.cpp to `52a939a2a762224e255d366c1182b2af4dd1a032` (#11877) :arrow_up: Update ggml-org/whisper.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/whisper/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/whisper/Makefile b/backend/go/whisper/Makefile index c1844ff6d..bd789a915 100644 --- a/backend/go/whisper/Makefile +++ b/backend/go/whisper/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # whisper.cpp version WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp -WHISPER_CPP_VERSION?=eacbd8234c6654cdbf2c377f72b2106875479bdc +WHISPER_CPP_VERSION?=52a939a2a762224e255d366c1182b2af4dd1a032 SO_TARGET?=libgowhisper.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From 3e98ec935a2da4fe91029c67becae2f89a94a032 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 01:03:02 +0200 Subject: [PATCH 10/85] chore(deps): bump grpcio from 1.82.1 to 1.83.1 in /backend/python/common/template (#11858) chore(deps): bump grpcio in /backend/python/common/template Bumps [grpcio](https://github.com/grpc/grpc) from 1.82.1 to 1.83.1. - [Release notes](https://github.com/grpc/grpc/releases) - [Commits](https://github.com/grpc/grpc/compare/v1.82.1...v1.83.1) --- updated-dependencies: - dependency-name: grpcio dependency-version: 1.83.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- backend/python/common/template/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/python/common/template/requirements.txt b/backend/python/common/template/requirements.txt index 1795a9694..205cdbb75 100644 --- a/backend/python/common/template/requirements.txt +++ b/backend/python/common/template/requirements.txt @@ -1,3 +1,3 @@ -grpcio==1.82.1 +grpcio==1.83.1 protobuf grpcio-tools \ No newline at end of file From f09d6092ed4122afa90ea47a0fda429940004718 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 5 Sep 2026 11:24:25 +0200 Subject: [PATCH 11/85] chore: :arrow_up: Update 0xShug0/audio.cpp to `cd98afb3264044aeff33618df486c81de1bee7e7` (#11878) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index dc4455c13..16bc49cae 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=8fe1ab805555c72aaf06d83828b523dfc6ed796d +AUDIO_CPP_VERSION?=cd98afb3264044aeff33618df486c81de1bee7e7 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 7ab7a3136bea263ffed10a7000b16bbd7d50b3ac Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 5 Sep 2026 11:25:24 +0200 Subject: [PATCH 12/85] chore(model gallery): :robot: add 1 new models via gallery agent (#11880) chore(model gallery): :robot: add new models via gallery agent Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index f716e60b7..94b8a1e56 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,59 @@ --- +- name: "qwopus3.8-27b-flash" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Jackrong/Qwopus3.8-27B-Flash-GGUF + description: | + # Qwen3.8-27B + + > [!Note] + > This repository contains model weights and configuration files for the post-trained model in the Hugging Face Transformers format. + > + > These artifacts are compatible with Hugging Face Transformers, vLLM, SGLang, TokenSpeed, etc. + + > [!Tip] + > For users seeking managed, scalable inference without infrastructure maintenance, the official Qwen API service is provided by Qwen Cloud. + > In particular, **Qwen3.8-27B** will be available as a hosted version with more production features, e.g., 1M context length by default, official built-in tools. For more information, please refer to the Qwen3.8-27B Overview. The service is coming soon. Stay tuned for updates. + + Following the widespread community adoption of the Qwen3.5 and Qwen3.6 series, we are pleased to introduce Qwen3.8, the most capable generation in the Qwen open-model family to date. + + ... + license: "apache-2.0" + tags: + - llm + - gguf + - qwen + - qwen3 + - vision + - multimodal + - instruction-tuned + - reasoning + icon: https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3.5/demo/CI_Demo/mathv-1327.jpg + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + mmproj: llama-cpp/mmproj/Qwopus3.8-27B-Flash-MTP-Q4_K_M/mmproj-F32.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:6 + - spec_p_min:0.75 + parameters: + model: llama-cpp/models/Qwopus3.8-27B-Flash-MTP-Q4_K_M/Qwopus3.8-27B-Flash-MTP-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Qwopus3.8-27B-Flash-MTP-Q4_K_M/Qwopus3.8-27B-Flash-MTP-Q4_K_M.gguf + sha256: 2b9a335bd33bb977d3127062061c12cc94f37ceef1850d372317fbf4ee89d9c5 + uri: https://huggingface.co/Jackrong/Qwopus3.8-27B-Flash-GGUF/resolve/main/Qwopus3.8-27B-Flash-MTP-Q4_K_M.gguf + - filename: llama-cpp/mmproj/Qwopus3.8-27B-Flash-MTP-Q4_K_M/mmproj-F32.gguf + sha256: be3f444ecaedf084fd19399ba18cefcbbe3789bb81f4ad5ed67c91a1a3f48a67 + uri: https://huggingface.co/Jackrong/Qwopus3.8-27B-Flash-GGUF/resolve/main/mmproj-F32.gguf - name: "glm-5.3" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From ee2920d6197dfd07974fee0f87dcde53dd95b55c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:25:54 +0200 Subject: [PATCH 13/85] chore(deps): bump the npm_and_yarn group across 1 directory with 3 updates (#11869) Bumps the npm_and_yarn group with 3 updates in the /core/http/react-ui directory: [@humanfs/node](https://github.com/humanwhocodes/humanfs/tree/HEAD/packages/node), [fast-uri](https://github.com/fastify/fast-uri) and [qs](https://github.com/ljharb/qs). Updates `@humanfs/node` from 0.16.7 to 0.16.8 - [Release notes](https://github.com/humanwhocodes/humanfs/releases) - [Changelog](https://github.com/humanwhocodes/humanfs/blob/main/packages/node/CHANGELOG.md) - [Commits](https://github.com/humanwhocodes/humanfs/commits/node-v0.16.8/packages/node) Updates `fast-uri` from 3.1.5 to 3.1.7 - [Release notes](https://github.com/fastify/fast-uri/releases) - [Commits](https://github.com/fastify/fast-uri/compare/v3.1.5...v3.1.7) Updates `qs` from 6.15.3 to 6.16.0 - [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/qs/compare/v6.15.3...v6.16.0) --- updated-dependencies: - dependency-name: "@humanfs/node" dependency-version: 0.16.8 dependency-type: indirect dependency-group: npm_and_yarn - dependency-name: fast-uri dependency-version: 3.1.7 dependency-type: indirect dependency-group: npm_and_yarn - dependency-name: qs dependency-version: 6.16.0 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- core/http/react-ui/package-lock.json | 103 +++++++++++++++++++-------- core/http/react-ui/package.json | 2 +- 2 files changed, 75 insertions(+), 30 deletions(-) diff --git a/core/http/react-ui/package-lock.json b/core/http/react-ui/package-lock.json index 9c8edbfc8..7ac92af0c 100644 --- a/core/http/react-ui/package-lock.json +++ b/core/http/react-ui/package-lock.json @@ -32,7 +32,7 @@ "react": "^19.1.0", "react-dom": "^19.1.0", "react-i18next": "^17.0.6", - "react-router": "7.18.2", + "react-router": "8.3.1", "react-router-dom": "7.18.2", "yaml": "^2.8.3" }, @@ -649,27 +649,43 @@ } }, "node_modules/@humanfs/core": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", - "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, "engines": { "node": ">=18.18.0" } }, "node_modules/@humanfs/node": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", - "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", "dev": true, + "license": "Apache-2.0", "dependencies": { - "@humanfs/core": "^0.19.1", + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", "@humanwhocodes/retry": "^0.4.0" }, "engines": { "node": ">=18.18.0" } }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", @@ -2198,6 +2214,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/cookie-es": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/cookie-es/-/cookie-es-3.1.1.tgz", + "integrity": "sha512-UaXxwISYJPTr9hwQxMFYZ7kNhSXboMXP+Z3TRX6f1/NyaGPfuNUZOWP1pUEb75B2HjfklIYLVRfWiFZJyC6Npg==", + "license": "MIT" + }, "node_modules/cookie-signature": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", @@ -2889,9 +2911,9 @@ "dev": true }, "node_modules/fast-uri": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", - "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", "funding": [ { "type": "github", @@ -5951,9 +5973,9 @@ } }, "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "license": "BSD-3-Clause", "dependencies": { "es-define-property": "^1.0.1", @@ -6061,22 +6083,24 @@ } }, "node_modules/react": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", - "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", + "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", + "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/react-dom": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", - "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", + "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", + "license": "MIT", "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { - "react": "^19.2.4" + "react": "^19.2.8" } }, "node_modules/react-i18next": { @@ -6106,20 +6130,19 @@ } }, "node_modules/react-router": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.2.tgz", - "integrity": "sha512-aUVMjFm3GAPTTZL7oYr5E7ETiqfQCHRLH+B+5afnICvf0r7kkK4eR6SMuwbSTJw/7t+12khT/Kahij49fqOCIg==", + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-8.3.1.tgz", + "integrity": "sha512-TEOpiO2g0TJHEOJeRVv4amUFun9v1npCKszvcquNvzETUtJ8udV86ah5eFoHT7g26bsBvT6EiIhqulR8eDF++A==", "license": "MIT", "dependencies": { - "cookie": "^1.0.1", - "set-cookie-parser": "^2.6.0" + "cookie-es": "^3.1.1" }, "engines": { - "node": ">=20.0.0" + "node": ">=22.22.0" }, "peerDependencies": { - "react": ">=18", - "react-dom": ">=18" + "react": ">=19.2.7", + "react-dom": ">=19.2.7" }, "peerDependenciesMeta": { "react-dom": { @@ -6143,6 +6166,28 @@ "react-dom": ">=18" } }, + "node_modules/react-router-dom/node_modules/react-router": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.2.tgz", + "integrity": "sha512-aUVMjFm3GAPTTZL7oYr5E7ETiqfQCHRLH+B+5afnICvf0r7kkK4eR6SMuwbSTJw/7t+12khT/Kahij49fqOCIg==", + "license": "MIT", + "dependencies": { + "cookie": "^1.0.1", + "set-cookie-parser": "^2.6.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", diff --git a/core/http/react-ui/package.json b/core/http/react-ui/package.json index fc3832680..9970ba108 100644 --- a/core/http/react-ui/package.json +++ b/core/http/react-ui/package.json @@ -47,7 +47,7 @@ "react": "^19.1.0", "react-dom": "^19.1.0", "react-i18next": "^17.0.6", - "react-router": "7.18.2", + "react-router": "8.3.1", "react-router-dom": "7.18.2", "yaml": "^2.8.3" }, From 7ad4384cbe91b22388ec65e972a528d8ce53d455 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 5 Sep 2026 11:26:11 +0200 Subject: [PATCH 14/85] chore: :arrow_up: Update ggml-org/llama.cpp to `427291b5b34cd914a31b3fd3b61a68f6184f4b9f` (#11866) :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index f6d8bd16f..35c965f2b 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=9cffdcc801582616250520966699cb5b25d28243 +LLAMA_VERSION?=427291b5b34cd914a31b3fd3b61a68f6184f4b9f LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From 32875a708983ea2ee2a1e0f5d68067e4ba25cf7a Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 5 Sep 2026 11:26:30 +0200 Subject: [PATCH 15/85] chore: :arrow_up: Update CrispStrobe/CrispASR to `c6d4265ae2ee2b8931b09d7d25d5c65c75c36a41` (#11862) :arrow_up: Update CrispStrobe/CrispASR Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/crispasr/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 50293f642..e01dbef0a 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=ff3945c94cab9191199a5d531a32c4e9535c094b +CRISPASR_VERSION?=c6d4265ae2ee2b8931b09d7d25d5c65c75c36a41 SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From b217148bbc229abe6dc079ce78d250e168e818e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:27:01 +0200 Subject: [PATCH 16/85] chore(deps): bump torch from 2.13.0+xpu to 2.14.0+xpu in /backend/python/common/template (#11859) chore(deps): bump torch in /backend/python/common/template Bumps torch from 2.13.0+xpu to 2.14.0+xpu. --- updated-dependencies: - dependency-name: torch dependency-version: 2.14.0+xpu dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- backend/python/common/template/requirements-intel.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/python/common/template/requirements-intel.txt b/backend/python/common/template/requirements-intel.txt index cfe9fd3fa..c17f22c30 100644 --- a/backend/python/common/template/requirements-intel.txt +++ b/backend/python/common/template/requirements-intel.txt @@ -1,4 +1,4 @@ --extra-index-url https://download.pytorch.org/whl/xpu -torch==2.13.0+xpu +torch==2.14.0+xpu oneccl_bind_pt==2.8.0+xpu optimum[openvino] \ No newline at end of file From a98501d6cedaf5b4e02ab6af92e2bb2cb8604fed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abdullah=20Mansour=20=7C=20=D8=B9=D8=A8=D8=AF=20=D8=A7?= =?UTF-8?q?=D9=84=D9=84=D9=87=20=D9=85=D9=86=D8=B5=D9=88=D8=B1?= Date: Sun, 6 Sep 2026 00:46:24 +0300 Subject: [PATCH 17/85] docs(llama-cpp): clarify multimodal speculative decoding (#11700) * docs(llama-cpp): clarify multimodal speculative decoding Update the speculative decoding guidance now that modern llama.cpp backends can combine mmproj-based vision with speculative decoding, including MTP. Document compatibility checks, draft acceptance statistics, VRAM tradeoffs, and a combined configuration example. Assisted-by: Codex:GPT-5.6-Sol [gh] [OpenStack] [Docker] Signed-off-by: Abdullah Mansour * docs(llama-cpp): clarify multimodal MTP references Distinguish the upstream change that removed the general multimodal speculative restriction from the later change that added MTP with explicit vision compatibility. Assisted-by: Codex:GPT-5.6-Sol [gh] [Docker] Signed-off-by: Abdullah Mansour --------- Signed-off-by: Abdullah Mansour --- docs/content/advanced/model-configuration.md | 23 ++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index 8e0ca855b..b748fca05 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -397,7 +397,9 @@ The canonical names match upstream llama.cpp (dash-separated). For backward comp Multiple types can be chained by passing a comma-separated list to `spec_type` (e.g. `spec_type:ngram-simple,ngram-mod`). The runtime tries them in order and accepts the first proposal that meets the acceptance criteria. {{% notice note %}} -Speculative decoding is automatically disabled when multimodal models (with `mmproj`) are active. The `n_draft` parameter can also be overridden per-request. +The current LocalAI llama.cpp backend supports speculative decoding with multimodal models that load an `mmproj`, including MTP. LocalAI passes both configurations to llama.cpp and does not disable speculation merely because an `mmproj` is present. Upstream llama.cpp removed the former general multimodal/speculative restriction in [ggml-org/llama.cpp#19493](https://github.com/ggml-org/llama.cpp/pull/19493); [ggml-org/llama.cpp#22673](https://github.com/ggml-org/llama.cpp/pull/22673) later added MTP support and explicitly documented its compatibility with vision input. + +Compatibility still depends on the installed backend version and the target/draft model architecture. Check the backend logs for successful projector loading and speculative-context initialization, then look for the `draft acceptance` statistics line and its `accepted / generated` counts. A representative run with zero accepted draft tokens receives no speculative speedup and can indicate that the model or settings need tuning. {{% /notice %}} ##### Multi-Token Prediction (MTP) @@ -427,7 +429,7 @@ Detection runs both at **import time** (the `/import-model` UI / `POST /models/i | `spec_type` | `draft-mtp` | Activates MTP. Can be chained with other types (see below). | | `spec_n_max` / `draft_max` | `2`-`6` | Number of draft tokens per step. Upstream's PR suggests 2-3 for the tightest acceptance window; LocalAI's auto-default is 6 to favour throughput on models with high acceptance. | | `spec_p_min` | `0.75` | Pinned because upstream marks the current default with a "change to 0.0f" TODO; locking it here keeps acceptance thresholds stable across future llama.cpp bumps. | -| `mmproj_use_gpu` | `false` (or unset `mmproj`) | MTP has a prompt-processing overhead; if the model is non-vision, drop the mmproj entirely to save VRAM. | +| `mmproj_use_gpu` | `true` for vision | MTP does not require disabling the projector. Keep `mmproj` configured for image input; set this option to `false` to keep the projector on CPU when VRAM is tight. Remove `mmproj` only for text-only use when vision is not needed. | **Minimal config** (override-only, since auto-detection already covers this for MTP-capable GGUFs): @@ -441,6 +443,23 @@ options: - spec_n_max:3 ``` +**With vision enabled:** + +```yaml +name: qwen3-vision-mtp +backend: llama-cpp +known_usecases: + - chat + - vision +parameters: + model: qwen3-with-mtp.gguf +mmproj: mmproj-qwen3.gguf +options: + - spec_type:draft-mtp + - spec_n_max:3 + - spec_p_min:0.75 +``` + **With a separate MTP head file:** ```yaml From 24f897cd096c604cf27a696b77ac321cb9908223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=BF=97=E8=B0=A6?= <89645338+simpleqt@users.noreply.github.com> Date: Sun, 6 Sep 2026 05:47:13 +0800 Subject: [PATCH 18/85] docs: fix dead anchors and a dead section link (#11885) - middleware.md: the 'default detector' link used #instance-wide-defaults; the heading is 'Instance-wide default detector' - the advanced/reference landing pages linked an ../installation/ directory that does not exist in docs/content; dropped the dead bullets (deployment content lives under getting-started) --- docs/content/advanced/_index.en.md | 3 +-- docs/content/operations/middleware.md | 2 +- docs/content/reference/_index.en.md | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/content/advanced/_index.en.md b/docs/content/advanced/_index.en.md index c81602ea1..8b0b4ece3 100644 --- a/docs/content/advanced/_index.en.md +++ b/docs/content/advanced/_index.en.md @@ -103,8 +103,7 @@ Before diving into advanced topics, ensure you have: ## Related Sections - 📚 [Reference](../reference/) - API documentation and command reference -- 🔌 [Installation](../installation/) - Deployment options and requirements -- ⭐ [Features](../features/) - Overview of LocalAI capabilities +- - ⭐ [Features](../features/) - Overview of LocalAI capabilities --- diff --git a/docs/content/operations/middleware.md b/docs/content/operations/middleware.md index fc2f2a3c9..43ab02dc1 100644 --- a/docs/content/operations/middleware.md +++ b/docs/content/operations/middleware.md @@ -45,7 +45,7 @@ routing, `/api/pii/events` for redaction and block actions. PII redaction is **NER-based and runs request-side (input)**. It is **off by default**, flipping to **on for any `cloud-proxy` backend** because that traffic crosses the network to a third-party provider. Pick a -[default detector](#instance-wide-defaults) so those models are actually +[default detector](#instance-wide-default-detector) so those models are actually scanned. Explicit `pii.enabled` in a model's YAML always wins over the backend default. diff --git a/docs/content/reference/_index.en.md b/docs/content/reference/_index.en.md index 654f8b02b..91b090938 100644 --- a/docs/content/reference/_index.en.md +++ b/docs/content/reference/_index.en.md @@ -168,8 +168,7 @@ Before using reference documentation, ensure you have: ## Related Sections - 📖 [Advanced](../advanced/) - Deep dive into configuration and optimization -- 🔌 [Installation](../installation/) - Setup and deployment -- ⭐ [Features](../features/) - Feature overview +- - ⭐ [Features](../features/) - Feature overview --- From d9d68c595eaa5f2dd2933026db4396958a91135f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=BF=97=E8=B0=A6?= <89645338+simpleqt@users.noreply.github.com> Date: Sun, 6 Sep 2026 05:48:01 +0800 Subject: [PATCH 19/85] docs: fix dead links in the PR template and backends docs (#11884) - the PR template's 'Signed commits' anchor pointed at a CONTRIBUTING section that does not exist; repointed at the Commit messages section - the longcat-video backend README linked a docs page that was never committed; replaced the dead link with plain text - formal-verification/README.md used ../../../ for five in-repo packages (escaping the repo root); fixed to ../ --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- backend/python/longcat-video/README.md | 2 +- formal-verification/README.md | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6d0e30e7a..a7ae060fb 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ This PR fixes # **Notes for Reviewers** -**[Signed commits](../CONTRIBUTING.md#signing-off-on-commits-developer-certificate-of-origin)** +**[Signed commits](../CONTRIBUTING.md#commit-messages)** - [ ] Yes, I signed my commits. - [ ] Documentation updated (docs/content/) for user-facing changes, or not applicable diff --git a/backend/python/longcat-video/README.md b/backend/python/longcat-video/README.md index 821de0130..e50a40f51 100644 --- a/backend/python/longcat-video/README.md +++ b/backend/python/longcat-video/README.md @@ -11,7 +11,7 @@ RPC. It supports: systems such as NVIDIA DGX Spark. Install the `longcat-video` or `longcat-video-avatar-1.5` recipe from the -LocalAI Model Gallery. See the [LongCat user guide](../../../docs/content/features/longcat-video.md) +LocalAI Model Gallery. LongCat video backend for Studio and API examples, hardware requirements, and manual configuration. The upstream source is pinned in `Makefile` and patched at build time. The diff --git a/formal-verification/README.md b/formal-verification/README.md index 6bd1da8b0..58bde7a1e 100644 --- a/formal-verification/README.md +++ b/formal-verification/README.md @@ -24,11 +24,11 @@ then the implementation is checked against them. | `fizzbee.sha256` | Pinned checksum(s) of the FizzBee release the gate uses (created on first `install-fizzbee.sh` run). | The implementations under test live in -[`core/http/endpoints/openai/respcoord`](../../../core/http/endpoints/openai/respcoord) (M3), -[`core/http/endpoints/openai/turncoord`](../../../core/http/endpoints/openai/turncoord) (M2), -[`core/http/endpoints/openai/conncoord`](../../../core/http/endpoints/openai/conncoord) (M1), -[`core/http/endpoints/openai/compactcoord`](../../../core/http/endpoints/openai/compactcoord) (M4), -and [`core/http/endpoints/openai/ttscoord`](../../../core/http/endpoints/openai/ttscoord) (M5). +[`core/http/endpoints/openai/respcoord`](../core/http/endpoints/openai/respcoord) (M3), +[`core/http/endpoints/openai/turncoord`](../core/http/endpoints/openai/turncoord) (M2), +[`core/http/endpoints/openai/conncoord`](../core/http/endpoints/openai/conncoord) (M1), +[`core/http/endpoints/openai/compactcoord`](../core/http/endpoints/openai/compactcoord) (M4), +and [`core/http/endpoints/openai/ttscoord`](../core/http/endpoints/openai/ttscoord) (M5). ## Running the realtime gate From dd1776a91ff5539ba1ec195e6c4b1cf8e29922b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=BF=97=E8=B0=A6?= <89645338+simpleqt@users.noreply.github.com> Date: Sun, 6 Sep 2026 05:49:10 +0800 Subject: [PATCH 20/85] docs: correct documented env var and CLI flag names (#11886) - api-errors.md documented LOCALAI_SUBTLEKEY_COMPARISON (missing the KEY underscore); the code defines LOCALAI_SUBTLE_KEY_COMPARISON, so the documented variable silently did nothing - cli-reference.md documented a --csrf flag / $LOCALAI_CSRF env that do not exist, with inverted semantics; the actual flag is --disable-csrf (LOCALAI_DISABLE_CSRF), 'Disable CSRF middleware (enabled by default)' --- docs/content/reference/api-errors.md | 2 +- docs/content/reference/cli-reference.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/reference/api-errors.md b/docs/content/reference/api-errors.md index 4591e0d00..9bd9dea02 100644 --- a/docs/content/reference/api-errors.md +++ b/docs/content/reference/api-errors.md @@ -417,4 +417,4 @@ fi |-------------------------------|------------------------------------------------| | `LOCALAI_API_KEY` | Comma-separated list of valid API keys | | `LOCALAI_OPAQUE_ERRORS` | Set to `true` to hide error details (returns empty body with status code only) | -| `LOCALAI_SUBTLEKEY_COMPARISON`| Use constant-time key comparison for timing-attack resistance | +| `LOCALAI_SUBTLE_KEY_COMPARISON`| Use constant-time key comparison for timing-attack resistance | diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index fb0872545..2ba6c96e8 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -85,7 +85,7 @@ For more information on VRAM management, see [VRAM and Memory Management]({{%rel | `--max-concurrent-backend-requests` | `1024` | Process-wide ceiling for concurrent backend inference operations. Excess inference receives HTTP 503 with `Retry-After`; UI and administrative endpoints remain available | `$LOCALAI_MAX_CONCURRENT_BACKEND_REQUESTS`, `$MAX_CONCURRENT_BACKEND_REQUESTS` | | `--cors` | `false` | Enable CORS (Cross-Origin Resource Sharing) | `$LOCALAI_CORS`, `$CORS` | | `--cors-allow-origins` | | Comma-separated list of allowed CORS origins | `$LOCALAI_CORS_ALLOW_ORIGINS`, `$CORS_ALLOW_ORIGINS` | -| `--csrf` | `false` | Enable Fiber CSRF middleware | `$LOCALAI_CSRF` | +| `--disable-csrf` | `false` | Disable CSRF middleware (enabled by default) | `$LOCALAI_DISABLE_CSRF` | | `--disable-http-compression` | `false` | Disable gzip compression of HTTP responses. Compression is enabled by default; streaming endpoints (streaming chat completions, SSE bridges, WebSocket upgrades) and already-compressed formats are never compressed | `$LOCALAI_DISABLE_HTTP_COMPRESSION` | | `--http-compression-min-length` | `1024` | Minimum response size in bytes before gzip compression is applied. Smaller responses are sent as-is because the gzip envelope would outweigh the saving | `$LOCALAI_HTTP_COMPRESSION_MIN_LENGTH` | | `--upload-limit` | `15` | Default upload-limit in MB | `$LOCALAI_UPLOAD_LIMIT`, `$UPLOAD_LIMIT` | From 7ec55db3277c8c7ce658b1a58418a21942165af9 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 11 Aug 2026 12:06:04 +0000 Subject: [PATCH 21/85] feat(gallery): add Supra2 100M Instruct Add the official F16 GGUF build of the compact Apache-2.0 Qwen3-family chat model with its native 2K context limit. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 94b8a1e56..c59f3dec8 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -159,6 +159,35 @@ - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00016-of-00016.gguf sha256: 79af2211278ac07dfe4c789751d7de7d7caf0a4516486f8a9b6eedb1f3fb6e9b uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00016-of-00016.gguf +- name: "supra2-100m-instruct" + url: "github:mudler/LocalAI/gallery/qwen3.yaml@master" + urls: + - https://huggingface.co/SupraLabs/Supra2-100M-Instruct + description: | + Supra2-100M-Instruct is a compact English chat model trained from scratch by + SupraLabs on the Qwen3 architecture. It has 100 million parameters, a + 2,048-token context window, and is intended for lightweight experiments and + constrained edge deployments. This entry uses the publisher's official F16 + GGUF build. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - chat + - qwen3 + - edge + - english + last_checked: "2026-08-11" + overrides: + parameters: + context_size: 2048 + model: Supra2-100M-SFT-F16.gguf + files: + - filename: Supra2-100M-SFT-F16.gguf + uri: huggingface://SupraLabs/Supra2-100M-Instruct/Supra2-100M-SFT-F16.gguf + sha256: f88228c3dcc13b5ee2333cbb6fc2416de908e5b8adabd39bf9ee4378f28451eb - &qwen3-8-flash-next name: "qwen3.8-flash-next-q4" variants: From 6a5895cd7fdf2b27b09b31597c0882bb63222277 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 13 Aug 2026 20:06:27 +0000 Subject: [PATCH 22/85] feat(gallery): add LightOnOCR 2 1B variants Add the compact LightOnOCR model to the OCR section with Q8_0 and F16 builds. Each build uses its matching vision projector. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 70 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index c59f3dec8..e61cd7eec 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -16540,6 +16540,76 @@ - filename: mmproj-Qwen3-Omni-30B-A3B-Thinking-Q8_0.gguf sha256: 2bd5459571f8230a0c251d3d0dd36267753f0800ed145449a34f220a31f93898 uri: huggingface://ggml-org/Qwen3-Omni-30B-A3B-Thinking-GGUF/mmproj-Qwen3-Omni-30B-A3B-Thinking-Q8_0.gguf +- &lightonocr-2-1b + name: "lightonocr-2-1b" + variants: + - model: lightonocr-2-1b-f16 + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/lightonai/LightOnOCR-2-1B + - https://huggingface.co/ggml-org/LightOnOCR-2-1B-GGUF + description: | + LightOnOCR-2-1B is a compact Apache-2.0 vision-language model for optical + character recognition and document understanding. It extracts text, + tables, forms, and structured content from images and PDFs in multiple + languages. This default entry uses the Q8_0 model and vision projector. + license: apache-2.0 + icon: https://huggingface.co/lightonai.png + tags: + - llm + - gguf + - gpu + - image-to-text + - ocr + - multimodal + - cpu + - mistral + last_checked: "2026-08-13" + overrides: + backend: llama-cpp + known_usecases: + - chat + - vision + mmproj: mmproj-LightOnOCR-2-1B-Q8_0.gguf + options: + - use_jinja:true + parameters: + model: LightOnOCR-2-1B-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: LightOnOCR-2-1B-Q8_0.gguf + sha256: f4cfc0ea9765a4cac22a235191c48aefa94df223419f5ab658b6201f36206d58 + uri: huggingface://ggml-org/LightOnOCR-2-1B-GGUF/LightOnOCR-2-1B-Q8_0.gguf + - filename: mmproj-LightOnOCR-2-1B-Q8_0.gguf + sha256: e9b45eb85b0f0afb3cc138236eec7adb75879420130e5e2049ee9af3f24e337f + uri: huggingface://ggml-org/LightOnOCR-2-1B-GGUF/mmproj-LightOnOCR-2-1B-Q8_0.gguf +- !!merge <<: *lightonocr-2-1b + name: "lightonocr-2-1b-f16" + variants: [] + description: | + LightOnOCR-2-1B F16 is the full-precision GGUF build for optical character + recognition and multilingual document understanding. It pairs the F16 + language model with the matching F16 vision projector. + overrides: + backend: llama-cpp + known_usecases: + - chat + - vision + mmproj: mmproj-LightOnOCR-2-1B-f16.gguf + options: + - use_jinja:true + parameters: + model: LightOnOCR-2-1B-f16.gguf + template: + use_tokenizer_template: true + files: + - filename: LightOnOCR-2-1B-f16.gguf + sha256: 83f5b7a24410de69390119dd18e7866ecd0687564423846fc344ed3a46647a0b + uri: huggingface://ggml-org/LightOnOCR-2-1B-GGUF/LightOnOCR-2-1B-f16.gguf + - filename: mmproj-LightOnOCR-2-1B-f16.gguf + sha256: 1c4a3a27f7c5aa90b455ed5ec6dc79c9b9746739a8d840022790414376738130 + uri: huggingface://ggml-org/LightOnOCR-2-1B-GGUF/mmproj-LightOnOCR-2-1B-f16.gguf - name: glm-ocr url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: From bb4432d2f2fb09eac8e021f1ad024f7d3168a17c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 15 Aug 2026 12:06:53 +0000 Subject: [PATCH 23/85] feat(gallery): add IndexTTS 2.5 Add the original-dtype GGUF for LocalAI\x27s audio.cpp backend. The entry exposes multilingual voice cloning and records the model license.\n\nAssisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index e61cd7eec..045b68419 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -55033,6 +55033,47 @@ - model: gemma-4-26b-a4b-it-heretic-apex-i-balanced - model: gemma-4-26b-a4b-it-heretic-apex-i-compact - model: gemma-4-26b-a4b-it-heretic-apex-i-mini +- name: audio-cpp-indextts-2.5 + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/IndexTeam/IndexTTS-2.5 + - https://huggingface.co/Richasy/IndexTTS-2.5-GGUF + - https://github.com/0xShug0/audio.cpp + description: | + IndexTTS 2.5 is a multilingual text-to-speech model with zero-shot voice + cloning and emotion control, served by the audio-cpp backend. Supply a + server-local WAV path in the OpenAI `voice` field as the speaker reference. + + This entry uses the published original-dtype GGUF. The package preserves + the source tensors without another quantization step, and audio.cpp records + the original-dtype IndexTTS2 path as passing its validation suite. + + The bilibili Model Use License governs the model. Review its use, + redistribution, and large-scale commercial deployment terms before + installation. + license: other + tags: + - audio-cpp + - indextts + - multilingual + - tts + - text-to-speech + - voice-cloning + - emotion-control + - gguf + - ggml + last_checked: "2026-08-15" + overrides: + backend: audio-cpp + known_usecases: + - tts + name: audio-cpp-indextts-2.5 + parameters: + model: audio-cpp/index-tts2_5-orig.gguf + files: + - filename: audio-cpp/index-tts2_5-orig.gguf + sha256: 07e9bfe77bd42b6e67b8e5a39b365635f7198951281990cb49f322ef546cb9b4 + uri: huggingface://Richasy/IndexTTS-2.5-GGUF/IndexTTS2.5-GGUF/index-tts2_5-orig.gguf - name: audio-cpp-supertonic url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: From cfafd6cd364074f10dea5460e7c74b6e74d0d1f4 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 19 Aug 2026 20:06:48 +0000 Subject: [PATCH 24/85] feat(gallery): add Qwen3.8 NVFP4 MTP variant Add a compact NVFP4 build with its embedded MTP head and vision projector. This gives Blackwell-capable llama.cpp hosts a speculative option alongside the existing Q4 and Q8 builds. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 65 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 045b68419..7d956d022 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1802,6 +1802,7 @@ name: "qwen3.8-27b-q4" variants: - model: qwen3.8-27b-q4-mtp + - model: qwen3.8-27b-nvfp4-mtp - model: qwen3.8-27b-ridge - model: qwen3.8-27b-gsq-rco-iq2-xs - model: qwen3.8-27b-gsq-rco-iq2-s @@ -1899,7 +1900,7 @@ known_usecases: - chat - vision - mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-Q8_0.gguf + mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-BF16.gguf options: - use_jinja:true - spec_type:draft-mtp @@ -1925,6 +1926,68 @@ - filename: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-Q8_0.gguf uri: huggingface://ggml-org/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf sha256: 2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb +- !!merge <<: *qwen3-8-27b + name: "qwen3.8-27b-nvfp4-mtp" + variants: [] + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/esatapedico/Qwen3.8-27B-NVFP4-MTP-GGUF + description: | + Qwen3.8-27B in a compact NVFP4 GGUF format with its MTP draft head + embedded in the model file. This entry uses the medium tier, which keeps + the NVFP4 backbone while using higher-precision output and embedding + tensors. MTP speculative decoding proposes multiple tokens for the target + model to verify. + tags: + - llm + - gguf + - gpu + - qwen + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + - nvfp4 + - mtp + - speculative + last_checked: "2026-08-19" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-Q8_0.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:6 + - spec_p_min:0.75 + parameters: + min_p: 0 + model: llama-cpp/models/qwen3.8-27b/Qwen3.8-27B-NVFP4-MTP-MEDIUM.gguf + presence_penalty: 0 + repeat_penalty: 1 + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-27b/Qwen3.8-27B-NVFP4-MTP-MEDIUM.gguf + uri: huggingface://esatapedico/Qwen3.8-27B-NVFP4-MTP-GGUF/Qwen3.8-27B-NVFP4-MTP-MEDIUM.gguf + sha256: f0b4c538c75037f026bde3b650f0ca639d382c128a4572769dce1183db86253a + - filename: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-BF16.gguf + uri: huggingface://esatapedico/Qwen3.8-27B-NVFP4-MTP-GGUF/mmproj-BF16.gguf + sha256: 83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53 - !!merge <<: *qwen3-8-27b name: "qwen3.8-27b-q8" variants: [] From e2ef7d050df633c899fd66799aca94ba24f4e11b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 20 Aug 2026 16:04:57 +0000 Subject: [PATCH 25/85] feat(gallery): add Qwen3.8 9B Distill variants Add the Q4_K_M and Q8_0 GGUF builds for a compact Qwen3.5-based reasoning and tool-use distillation. The variant pair lets LocalAI choose higher fidelity when the host has enough memory while keeping the Q4 build as the installable fallback. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 79 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 7d956d022..b118091b4 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -3217,6 +3217,85 @@ - filename: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q8_0.gguf uri: huggingface://empero-ai/Qwen3.8-4B-Distill-GGUF/Qwen3.8-4B-Q8_0.gguf sha256: 770b780d6754a4954d1caf395c9239eaeb394f15c7a7ea34039883377c93c9c3 +- &qwen3-8-9b-distill + name: "qwen3.8-9b-distill-q4" + variants: + - model: qwen3.8-9b-distill-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/empero-ai/Qwen3.8-9B-Distill + - https://huggingface.co/empero-ai/Qwen3.8-9B-Distill-GGUF + description: | + Qwen3.8 9B Distill is an Apache-2.0, text-only Qwen3.5 9B fine-tune + distilled from Qwen3.8 2.4T A95B reasoning traces. It targets mathematics, + coding, instruction following, and function calling with a 262K native + context window. This entry uses the balanced Q4_K_M GGUF quantization; the + Q8_0 variant offers higher fidelity. + license: apache-2.0 + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + tags: + - llm + - gguf + - cpu + - gpu + - qwen3.5 + - reasoning + - coding + - tool-use + last_checked: "2026-08-20" + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q4_K_M.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q4_K_M.gguf + uri: huggingface://empero-ai/Qwen3.8-9B-Distill-GGUF/Qwen3.8-9B-Q4_K_M.gguf + sha256: df13d66021cef676f82be74053220fd75af6bf2a6a7fb77f5222ab9e50744a7a +- !!merge <<: *qwen3-8-9b-distill + name: "qwen3.8-9b-distill-q8" + variants: [] + description: | + Qwen3.8 9B Distill in the higher-fidelity Q8_0 GGUF format. This text-only + Qwen3.5 9B fine-tune targets reasoning, coding, instruction following, and + function calling with a 262K native context window. + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - reasoning_format:deepseek + parameters: + model: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q8_0.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q8_0.gguf + uri: huggingface://empero-ai/Qwen3.8-9B-Distill-GGUF/Qwen3.8-9B-Q8_0.gguf + sha256: 79ca5d342a07922f2bbf38c8d892a79a3c8620c65feaf4b1c66b7830ae724db8 - name: "btl-4-compact" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From 6324ee1dd914eeb5ecfc2d695987d9002b99b039 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 22 Aug 2026 00:05:23 +0000 Subject: [PATCH 26/85] feat(gallery): add VoxCPM2 Add the artifact-backed VoxCPM2 model for multilingual speech synthesis, voice design, and controllable voice cloning. Assisted-by: Codex:gpt-5.6 [Codex] Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index b118091b4..3e34948d3 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -13091,6 +13091,36 @@ source: type: huggingface repo: openbmb/VoxCPM1.5 +- name: voxcpm2 + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/openbmb/VoxCPM2 + description: | + VoxCPM2 is a 2B-parameter text-to-speech model supporting 30 languages and 48 kHz output, with voice design and controllable voice cloning. + license: apache-2.0 + icon: https://cdn-avatars.huggingface.co/v1/production/uploads/1670387859384-633fe7784b362488336bbfad.png + tags: + - tts + - text-to-speech + - voice-cloning + - voice-design + - cpu + - gpu + last_checked: "2026-08-22" + overrides: + backend: voxcpm + known_usecases: + - tts + tts: + voice_cloning: true + parameters: + model: openbmb/VoxCPM2 + artifacts: + - name: model + target: model + source: + type: huggingface + repo: openbmb/VoxCPM2 - name: neutts-air url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: From c96904ddf4b8bffb3fdec0784cfe744df4da03fd Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 25 Aug 2026 00:04:38 +0000 Subject: [PATCH 27/85] feat(gallery): add DFM Mimir for vLLM DFM Mimir is a new permissively licensed Danish and English HRM-Text model. LocalAI already ships a vLLM version with native support for its architecture. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 3e34948d3..e31ae2868 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -393,6 +393,35 @@ - embeddings parameters: model: tencent/WeMM-Embedding-9B +- name: "dfm-mimir:vllm" + url: github:mudler/LocalAI/gallery/vllm.yaml@master + urls: + - https://huggingface.co/danish-foundation-models/DFM-Mimir + description: | + DFM Mimir is an Apache-2.0, instruction-tuned HRM-Text model from Danish + Foundation Models. It has about 1 billion parameters and a 4,096-token + context window. The model focuses on Danish and English chat, reasoning, + mathematics, and code generation, and uses only permissible post-training + data. This entry serves the official BF16 safetensors checkpoint with vLLM. + license: apache-2.0 + tags: + - llm + - chat + - reasoning + - coding + - instruction-tuned + - danish + - english + - hrm-text + - safetensors + - vllm + - gpu + last_checked: "2026-08-25" + overrides: + context_size: 4096 + parameters: + max_tokens: 4096 + model: danish-foundation-models/DFM-Mimir - &granite-4-2-3b name: "granite-4.2-3b-q4" variants: From ca3b5dad8b82830afc897fcb2c3813df160d068e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 26 Aug 2026 16:05:35 +0000 Subject: [PATCH 28/85] feat(gallery): add Laguna S APEX variants The new importance-matrix builds reduce Laguna S 2.1 from the existing 96 GB default to 73.9 GB or 54.4 GB. Add both as selectable llama.cpp variants. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index e31ae2868..c4345ab3e 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -6512,6 +6512,8 @@ variants: - model: laguna-s-2.1-q8 - model: laguna-s-2.1-dflash + - model: laguna-s-2.1-apex-i-quality + - model: laguna-s-2.1-apex-i-compact description: | Laguna S 2.1 is Poolside's 118B-parameter, 8B-active Mixture-of-Experts model for agentic software engineering. It supports tool use and a native one-million-token context window; the official GGUF recommends 256K context for best output quality. This default entry uses the current 96 GB Q4_K_M artifact, with imatrix-quantized routed experts and a Q8_0 signal path. @@ -6535,6 +6537,70 @@ - filename: llama-cpp/models/Laguna-S-2.1-GGUF/laguna-s-2.1-Q4_K_M.gguf sha256: a8b55c75714ea73fd90ec85de5defdc0b8d88ca0ad2108343cdd8fc22f7583e4 uri: https://huggingface.co/poolside/Laguna-S-2.1-GGUF/resolve/main/laguna-s-2.1-Q4_K_M.gguf +- !!merge <<: *laguna-s-2-1-q8 + name: "laguna-s-2.1-apex-i-quality" + variants: [] + urls: + - https://huggingface.co/poolside/Laguna-S-2.1 + - https://huggingface.co/Myric/Laguna-S-2.1-APEX-GGUF + description: | + Laguna S 2.1 in the 73.9 GB APEX-I Quality format. This community build + uses an importance matrix and mixed precision to preserve the always-active + signal path while reducing the memory required by the routed experts. + + License: OpenMDW 1.1. + last_checked: "2026-08-26" + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-quality-v2.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-quality-v2.gguf + sha256: 772886614b2f11ba62ea2c2ebb5d02804e1ab829329b50f644e83e174decfaf2 + uri: huggingface://Myric/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-quality-v2.gguf +- !!merge <<: *laguna-s-2-1-q8 + name: "laguna-s-2.1-apex-i-compact" + variants: [] + urls: + - https://huggingface.co/poolside/Laguna-S-2.1 + - https://huggingface.co/Myric/Laguna-S-2.1-APEX-GGUF + description: | + Laguna S 2.1 in the smaller 54.4 GB APEX-I Compact format. This community + build uses an importance matrix and mixed precision to reduce memory use + while retaining higher precision for the always-active signal path. + + License: OpenMDW 1.1. + last_checked: "2026-08-26" + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-compact-v2.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-compact-v2.gguf + sha256: b5964de8c5de37c9d4fcbd6894a68a95a81a2c404b12897d9581fc0d934134de + uri: huggingface://Myric/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-compact-v2.gguf - !!merge <<: *laguna-s-2-1-q8 name: "laguna-s-2.1-dflash" description: | From 4c1002cc780c053e6379293d0521665a60e8b871 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 27 Aug 2026 04:05:54 +0000 Subject: [PATCH 29/85] feat(gallery): add mxbai embed large variants Add Q4_K_M and Q8_0 llama.cpp builds plus the official F16 GGUF for Mixedbread mxbai-embed-large-v1. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 74 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index c4345ab3e..6c63b4d7f 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -3374,6 +3374,80 @@ - filename: llama-cpp/models/BTL-4-Compact/BTL-4-IQ2_XXS.gguf uri: huggingface://badtheorylabs/BTL-4-Compact/BTL-4-IQ2_XXS.gguf sha256: 6b7c298cf909fc04428ecf360a29dcc578188b1c90aa6ed435159f5a0d351496 +- &mxbai-embed-large-v1 + name: "mxbai-embed-large-v1-q4" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + variants: + - model: mxbai-embed-large-v1-q8 + - model: mxbai-embed-large-v1-f16 + license: apache-2.0 + urls: + - https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1 + - https://huggingface.co/ChristianAzinn/mxbai-embed-large-v1-gguf + description: | + Mixedbread's mxbai-embed-large-v1 is a 335M-parameter English BERT + embedding model for retrieval, semantic search, and RAG. It produces + 1,024-dimensional embeddings and supports sequences up to 512 tokens. + Prefix retrieval queries with `Represent this sentence for searching + relevant passages: `. This entry uses the balanced Q4_K_M GGUF. + tags: + - embeddings + - retrieval + - rag + - gguf + - cpu + - gpu + - english + last_checked: "2026-08-27" + overrides: + backend: llama-cpp + embeddings: true + known_usecases: + - embeddings + parameters: + model: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1.Q4_K_M.gguf + files: + - filename: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1.Q4_K_M.gguf + uri: huggingface://ChristianAzinn/mxbai-embed-large-v1-gguf/mxbai-embed-large-v1.Q4_K_M.gguf + sha256: 3869d462819e3f6cd2c1b0f8d6817e95cc1ed31fc09388432669209d1c6f1b65 +- !!merge <<: *mxbai-embed-large-v1 + name: "mxbai-embed-large-v1-q8" + variants: [] + description: | + Mixedbread's mxbai-embed-large-v1 in the higher-fidelity Q8_0 GGUF + format. This 335M-parameter English BERT model produces + 1,024-dimensional embeddings for retrieval, semantic search, and RAG. + overrides: + backend: llama-cpp + embeddings: true + known_usecases: + - embeddings + parameters: + model: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1.Q8_0.gguf + files: + - filename: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1.Q8_0.gguf + uri: huggingface://ChristianAzinn/mxbai-embed-large-v1-gguf/mxbai-embed-large-v1.Q8_0.gguf + sha256: bcdebca12aa16c0e51d166d97e4776efd46f905952b0c9acb968976eba2619f3 +- !!merge <<: *mxbai-embed-large-v1 + name: "mxbai-embed-large-v1-f16" + variants: [] + urls: + - https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1 + description: | + Mixedbread's mxbai-embed-large-v1 in the official full-precision F16 + GGUF format. This 335M-parameter English BERT model produces + 1,024-dimensional embeddings for retrieval, semantic search, and RAG. + overrides: + backend: llama-cpp + embeddings: true + known_usecases: + - embeddings + parameters: + model: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1-f16.gguf + files: + - filename: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1-f16.gguf + uri: huggingface://mixedbread-ai/mxbai-embed-large-v1/gguf/mxbai-embed-large-v1-f16.gguf + sha256: 819c2adf5ce6df2b6bd2ae4ca90d2a69f060afeb438d0c171db57daa02e39c3d - &nemotron-3-embed-1b name: "nemotron-3-embed-1b-q4" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" From 5f783f97a32e2b6984e5bfc09249886ebeb789a7 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 00:05:55 +0000 Subject: [PATCH 30/85] feat(gallery): add Security-SLM Gemma 4 Add the text-only Q4_K_M build for private red-team, blue-team, and security operations workloads. Configure the supported Gemma 4 model for llama.cpp with its verified Hugging Face checksum. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 6c63b4d7f..8a346d23b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -8577,6 +8577,51 @@ - filename: llama-cpp/mmproj/gemma-4-12B-it-qat-q4_0-gguf/mmproj-gemma-4-12b-it-qat-q4_0.gguf uri: https://huggingface.co/google/gemma-4-12B-it-qat-q4_0-gguf/resolve/main/mmproj-gemma-4-12b-it-qat-q4_0.gguf sha256: cb018338a7538a9814d994bfe54644c71eb7ed54e31eae2f721e45fd3c260da7 +- name: "security-slm-gemma-4-e2b-it-q4" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/entrick/Security-SLM-Gemma-4-E2B-it-GGUF + - https://huggingface.co/unsloth/gemma-4-E2B-it-unsloth-bnb-4bit + description: | + Security-SLM is a compact Gemma 4 E2B fine-tune for authorized red-team, + blue-team, security operations, and AI security work. It is designed for + private and air-gapped deployments where prompts can contain sensitive + incident data, policies, or source code. + + This entry uses the text-only 3.43 GB Q4_K_M GGUF release. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - gemma + - cybersecurity + - tools + last_checked: "2026-08-31" + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/security-slm-gemma-4-e2b-it/security-gemma-4-e2b-it.Q4_K_M.gguf + repeat_penalty: 1 + temperature: 1 + top_k: 64 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/security-slm-gemma-4-e2b-it/security-gemma-4-e2b-it.Q4_K_M.gguf + uri: huggingface://entrick/Security-SLM-Gemma-4-E2B-it-GGUF/security-gemma-4-e2b-it.Q4_K_M.gguf + sha256: 9046894db49c38390088d151dd73397ce1a948480862b3fb8fcbb1b9e977ee5e - name: "gemma-4-e2b-it-qat-q4_0" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From f4bbc4a7257368e30390fa74f376ea2c97e1cc59 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 20:06:49 +0000 Subject: [PATCH 31/85] feat(gallery): add Dirk Qwen3.8 variants Add Q4_K_XL and Q8_K_XL llama.cpp builds with the shared vision projector. Enable the preserved MTP head for speculative decoding. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 103 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 8a346d23b..b406efcf8 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -633,6 +633,109 @@ - filename: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q8_0.gguf uri: huggingface://ibm-granite/granite-4.2-30b-GGUF/granite-4.2-30b-Q8_0.gguf sha256: 005b0933353e9ba219b26e2667705bdb8dbc74eb50e4a4e6cb70fca108710f81 +- &dirk-qwen3-8-27b + name: "dirk-qwen3.8-27b-q4" + variants: + - model: dirk-qwen3.8-27b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF + description: | + Dirk is a Qwen3.8 27B vision-language model with a concise chat template + for agentic coding, reasoning, tool use, and general knowledge tasks. It + preserves the model's MTP head for speculative decoding and supports a + 262K-token context window. + + This default entry uses the Q4_K_XL GGUF and F16 vision projector. A + higher-quality Q8_K_XL build is available as a variant. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + - mtp + last_checked: "2026-08-31" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/dirk-qwen3.8-27b/mmproj-F16.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:6 + - spec_p_min:0.75 + parameters: + min_p: 0 + model: llama-cpp/models/dirk-qwen3.8-27b/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/dirk-qwen3.8-27b/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf + uri: huggingface://peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf + sha256: d1ad2472a147caa1111bae5ec710331dc50692d62ebbdb3fbc54d421c4e209bc + - filename: llama-cpp/mmproj/dirk-qwen3.8-27b/mmproj-F16.gguf + uri: huggingface://peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF/mmproj-F16.gguf + sha256: cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e +- !!merge <<: *dirk-qwen3-8-27b + name: "dirk-qwen3.8-27b-q8" + variants: [] + description: | + Dirk in the higher-quality Q8_K_XL GGUF format, with MTP speculative + decoding and the shared F16 vision projector for multimodal prompts. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/dirk-qwen3.8-27b/mmproj-F16.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:6 + - spec_p_min:0.75 + parameters: + min_p: 0 + model: llama-cpp/models/dirk-qwen3.8-27b/Dirk-Qwen3.8-27B-UD-Q8_K_XL.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/dirk-qwen3.8-27b/Dirk-Qwen3.8-27B-UD-Q8_K_XL.gguf + uri: huggingface://peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF/Dirk-Qwen3.8-27B-UD-Q8_K_XL.gguf + sha256: be2f08a260021fb471f91f2c5a52aa4afe143688e8b412beb5a36cad2d47c4cd + - filename: llama-cpp/mmproj/dirk-qwen3.8-27b/mmproj-F16.gguf + uri: huggingface://peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF/mmproj-F16.gguf + sha256: cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e - name: "qwen3.8-27b-dflash2" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From b2fdbdd52a0ba067f2a44131b64362fd30ed0fba Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 5 Sep 2026 21:59:17 +0000 Subject: [PATCH 32/85] feat(gallery): add LLM-jp 4 thinking LLM-jp 4 provides a recent Japanese and English reasoning model on the supported Llama architecture. Add its official Q4 and BF16 GGUF builds so hosts can select the fidelity that fits. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- gallery/index.yaml | 75 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index b406efcf8..d253ff4d0 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -188,6 +188,81 @@ - filename: Supra2-100M-SFT-F16.gguf uri: huggingface://SupraLabs/Supra2-100M-Instruct/Supra2-100M-SFT-F16.gguf sha256: f88228c3dcc13b5ee2333cbb6fc2416de908e5b8adabd39bf9ee4378f28451eb +- &llm-jp-4-33b-thinking + name: "llm-jp-4-33b-thinking-q4" + variants: + - model: llm-jp-4-33b-thinking-bf16 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/llm-jp/llm-jp-4-33b-thinking + - https://huggingface.co/llm-jp/llm-jp-4-33b-thinking-gguf + description: | + LLM-jp-4-33B-thinking is an Apache-2.0 Japanese and English reasoning + model from Japan's National Institute of Informatics. Its dense Llama + architecture has 33 billion parameters and a 65K-token context window. + The model was aligned with supervised fine-tuning and DPO for multi-turn + conversation and instruction following. + + This default entry uses the 20.2 GB Q4_K_M GGUF. The official 66.4 GB + BF16 weights are available as a higher-fidelity variant. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - multilingual + - japanese + - reasoning + - thinking + - tools + - long-context + last_checked: "2026-08-23" + overrides: + backend: llama-cpp + context_size: 65536 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/llm-jp-4-33b-thinking/llm-jp-4-33b-thinking-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/llm-jp-4-33b-thinking/llm-jp-4-33b-thinking-Q4_K_M.gguf + uri: huggingface://llm-jp/llm-jp-4-33b-thinking-gguf/llm-jp-4-33b-thinking-Q4_K_M.gguf + sha256: 9e48892c0d5ec256d05fc3258c9e50738852c39636fbb9fe1f4b1cde2f5e7520 +- !!merge <<: *llm-jp-4-33b-thinking + name: "llm-jp-4-33b-thinking-bf16" + variants: [] + description: | + LLM-jp-4-33B-thinking in the official 66.4 GB BF16 GGUF format. This + variant preserves the original model precision for hosts with enough + memory. + overrides: + backend: llama-cpp + context_size: 65536 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/llm-jp-4-33b-thinking/llm-jp-4-33b-thinking-BF16.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/llm-jp-4-33b-thinking/llm-jp-4-33b-thinking-BF16.gguf + uri: huggingface://llm-jp/llm-jp-4-33b-thinking-gguf/llm-jp-4-33b-thinking-BF16.gguf + sha256: 7bb8465702b5c4a5d94e03e62921d5917181a2eda56da31edfdcd47fab2e9964 - &qwen3-8-flash-next name: "qwen3.8-flash-next-q4" variants: From 9ca516d6efb49bbe7e21478948e1e72b6c949025 Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:59:22 +0000 Subject: [PATCH 33/85] fix(nix): use legacy-peer-deps for react-ui importNpmLock turns the same-version hono override into a file: tarball that conflicts with the direct dependency (EOVERRIDE). Pass --legacy-peer-deps so the flake build can proceed without touching package.json (open #11633). Fixes #11804 Signed-off-by: lei_lei <96427312+leilei3167@users.noreply.github.com> --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index 0f632e2d8..ddcdeb26c 100644 --- a/flake.nix +++ b/flake.nix @@ -18,6 +18,9 @@ npmRoot = ./core/http/react-ui; }; npmConfigHook = pkgs.importNpmLock.npmConfigHook; + # Avoid EOVERRIDE when importNpmLock rewrites the same-version hono + # override to a file: tarball that conflicts with the direct dependency. + npmFlags = [ "--legacy-peer-deps" ]; npmBuildScript = "build"; installPhase = '' From 8744de44d4e4226b51e6b110b0e936115fae2410 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 14 Aug 2026 20:04:04 +0000 Subject: [PATCH 34/85] fix(whisperx): reject unconfigured diarization WhisperX silently returned a plain transcript when diarization lacked the Hugging Face token required to load pyannote. Reject that request clearly so callers do not mistake missing speaker labels for a successful diarization. Convert WhisperX seconds to the nanosecond duration unit used by the transcription API. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --- backend/python/whisperx/backend.py | 10 ++++++-- .../python/whisperx/test_transcript_utils.py | 25 +++++++++++++++++++ backend/python/whisperx/transcript_utils.py | 12 +++++++++ docs/content/features/audio-to-text.md | 3 ++- 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 backend/python/whisperx/test_transcript_utils.py create mode 100644 backend/python/whisperx/transcript_utils.py diff --git a/backend/python/whisperx/backend.py b/backend/python/whisperx/backend.py index 7318e10b7..dc6202286 100644 --- a/backend/python/whisperx/backend.py +++ b/backend/python/whisperx/backend.py @@ -16,6 +16,7 @@ import grpc sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) from grpc_auth import get_auth_interceptors +from transcript_utils import require_diarization_token, seconds_to_nanoseconds @@ -81,6 +82,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): import whisperx from whisperx.diarize import DiarizationPipeline + try: + require_diarization_token(request.diarize, self.hf_token) + except ValueError as err: + context.abort(grpc.StatusCode.FAILED_PRECONDITION, str(err)) + resultSegments = [] text = "" try: @@ -117,8 +123,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # Build result segments for idx, seg in enumerate(transcript["segments"]): seg_text = seg.get("text", "") - start = int(seg.get("start", 0)) - end = int(seg.get("end", 0)) + start = seconds_to_nanoseconds(seg.get("start", 0)) + end = seconds_to_nanoseconds(seg.get("end", 0)) speaker = seg.get("speaker", "") resultSegments.append(backend_pb2.TranscriptSegment( diff --git a/backend/python/whisperx/test_transcript_utils.py b/backend/python/whisperx/test_transcript_utils.py new file mode 100644 index 000000000..debe2ea6e --- /dev/null +++ b/backend/python/whisperx/test_transcript_utils.py @@ -0,0 +1,25 @@ +import unittest + +import transcript_utils + + +class TestTranscriptUtils(unittest.TestCase): + def test_diarization_requires_hugging_face_token(self): + with self.assertRaisesRegex( + ValueError, + "HF_TOKEN is required for WhisperX diarization", + ): + transcript_utils.require_diarization_token(True, None) + + def test_diarization_does_not_require_token_when_disabled(self): + transcript_utils.require_diarization_token(False, None) + + def test_seconds_are_serialized_as_nanoseconds(self): + self.assertEqual( + transcript_utils.seconds_to_nanoseconds(3.25), + 3_250_000_000, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/python/whisperx/transcript_utils.py b/backend/python/whisperx/transcript_utils.py new file mode 100644 index 000000000..a8ac57510 --- /dev/null +++ b/backend/python/whisperx/transcript_utils.py @@ -0,0 +1,12 @@ +"""Helpers for WhisperX transcript responses.""" + + +def require_diarization_token(diarize, token): + """Reject diarization when WhisperX cannot load its gated pipeline.""" + if diarize and not token: + raise ValueError("HF_TOKEN is required for WhisperX diarization") + + +def seconds_to_nanoseconds(seconds): + """Convert WhisperX timestamps to the duration unit used by LocalAI.""" + return int(seconds * 1_000_000_000) diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 0312d392a..5a5e833cf 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -11,6 +11,7 @@ The transcription endpoint allows to convert audio files to text. The endpoint s - **[whisper.cpp](https://github.com/ggerganov/whisper.cpp)**: A C++ library for audio transcription (default) - **moonshine**: Ultra-fast transcription engine optimized for low-end devices - **faster-whisper**: Fast Whisper implementation with CTranslate2 +- **WhisperX**: Whisper transcription with word alignment and optional speaker diarization. Set `HF_TOKEN` and pass `diarize=true` to load WhisperX's gated pyannote diarization pipeline. - **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events). - **llama-cpp**: Route transcription to any multimodal-audio GGUF model served by the `llama-cpp` backend (e.g. [Qwen3-ASR](https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF), Voxtral, Qwen2-Audio). Under the hood the request is converted into a chat completion with the audio attached via the model's audio encoder - the same path the upstream llama.cpp server uses. Set `backend: llama-cpp` in the model YAML and point `mmproj` at the matching audio encoder. - **voxtral**: Voxtral-family models served by a dedicated backend @@ -109,7 +110,7 @@ In addition to `file` and `model`, the endpoint accepts the following multipart | `timestamp_granularities[]` | Multi-value form field: `word` and/or `segment`. Honored when the backend produces the requested granularity. | | `response_format` | One of `json` (default for backwards-compat), `verbose_json`, `text`, `srt`, `vtt`, `lrc`. | | `stream` | When `true`, the endpoint emits an SSE stream of `transcript.text.delta` events followed by a final `transcript.text.done` event. | -| `diarize` | LocalAI extension - speaker diarization (whisper.cpp only). | +| `diarize` | LocalAI extension - speaker diarization. WhisperX requires `HF_TOKEN`; requests fail with `FailedPrecondition` when it is missing. | The response body for `verbose_json` includes `text`, `language`, `duration`, and `segments[]` (with `speaker` populated when diarization is enabled). From ab3f7fc9045c2080d499410640da1b7c36825cbd Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 15 Aug 2026 16:06:21 +0000 Subject: [PATCH 35/85] fix(qwen-asr): select Intel XPU devices The Intel backend installs PyTorch XPU wheels, but Qwen ASR only checked CUDA and MPS. Every Intel model therefore loaded on the CPU. Select XPU when available and place the model on xpu:0. Keep the existing CUDA, MPS, and CPU placement behavior. Assisted-by: Codex:GPT-5 [apply_patch] [gh] Signed-off-by: Ettore Di Giacinto --- backend/python/qwen-asr/backend.py | 20 ++----- backend/python/qwen-asr/device_utils.py | 18 ++++++ backend/python/qwen-asr/device_utils_test.py | 58 ++++++++++++++++++++ 3 files changed, 81 insertions(+), 15 deletions(-) create mode 100644 backend/python/qwen-asr/device_utils.py create mode 100644 backend/python/qwen-asr/device_utils_test.py diff --git a/backend/python/qwen-asr/backend.py b/backend/python/qwen-asr/backend.py index ea5e877f2..a559ff5e6 100644 --- a/backend/python/qwen-asr/backend.py +++ b/backend/python/qwen-asr/backend.py @@ -18,6 +18,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) from grpc_auth import get_auth_interceptors from model_utils import resolve_model_reference +from device_utils import device_map_for, select_device @@ -95,13 +96,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): return backend_pb2.Reply(message=bytes("OK", 'utf-8')) def LoadModel(self, request, context): - if torch.cuda.is_available(): - device = "cuda" - else: - device = "cpu" - mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available() - if mps_available: - device = "mps" + device = select_device(torch) if not torch.cuda.is_available() and request.CUDA: return backend_pb2.Result(success=False, message="CUDA is not available") @@ -123,7 +118,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): model_path, local_only = resolve_model_reference( request, "Qwen/Qwen3-ASR-1.7B" ) - default_dtype = torch.bfloat16 if self.device == "cuda" else torch.float32 + default_dtype = torch.bfloat16 if self.device in ("cuda", "xpu") else torch.float32 load_dtype = default_dtype if "torch_dtype" in self.options: d = str(self.options["torch_dtype"]).lower() @@ -145,12 +140,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): if attn_implementation is not None and isinstance(attn_implementation, str): attn_implementation = attn_implementation.strip() or None - if self.device == "mps": - device_map = None - elif self.device == "cuda": - device_map = "cuda:0" - else: - device_map = "cpu" + device_map = device_map_for(self.device) load_kwargs = dict( dtype=load_dtype, @@ -423,4 +413,4 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run the gRPC server.") parser.add_argument("--addr", default="localhost:50051", help="The address to bind the server to.") args = parser.parse_args() - serve(args.addr) \ No newline at end of file + serve(args.addr) diff --git a/backend/python/qwen-asr/device_utils.py b/backend/python/qwen-asr/device_utils.py new file mode 100644 index 000000000..0e1cb8005 --- /dev/null +++ b/backend/python/qwen-asr/device_utils.py @@ -0,0 +1,18 @@ +def select_device(torch_module): + mps = getattr(getattr(torch_module, "backends", None), "mps", None) + if mps is not None and mps.is_available(): + return "mps" + if torch_module.cuda.is_available(): + return "cuda" + xpu = getattr(torch_module, "xpu", None) + if xpu is not None and xpu.is_available(): + return "xpu" + return "cpu" + + +def device_map_for(device): + if device == "mps": + return None + if device in ("cuda", "xpu"): + return f"{device}:0" + return "cpu" diff --git a/backend/python/qwen-asr/device_utils_test.py b/backend/python/qwen-asr/device_utils_test.py new file mode 100644 index 000000000..fde07f32b --- /dev/null +++ b/backend/python/qwen-asr/device_utils_test.py @@ -0,0 +1,58 @@ +import unittest + +from device_utils import device_map_for, select_device + + +class Availability: + def __init__(self, available): + self._available = available + + def is_available(self): + return self._available + + +class TorchStub: + def __init__(self, *, cuda=False, mps=False, xpu=False): + self.cuda = Availability(cuda) + self.backends = type("Backends", (), {"mps": Availability(mps)})() + self.xpu = Availability(xpu) + + +class SelectDeviceTest(unittest.TestCase): + def test_preserves_cuda_selection(self): + torch_module = TorchStub(cuda=True) + + self.assertEqual(select_device(torch_module), "cuda") + + def test_preserves_mps_selection(self): + torch_module = TorchStub(mps=True) + + self.assertEqual(select_device(torch_module), "mps") + + def test_selects_xpu_when_intel_gpu_is_available(self): + torch_module = TorchStub(xpu=True) + + self.assertEqual(select_device(torch_module), "xpu") + + def test_falls_back_to_cpu(self): + torch_module = TorchStub() + + self.assertEqual(select_device(torch_module), "cpu") + + +class DeviceMapTest(unittest.TestCase): + def test_preserves_cuda_model_placement(self): + self.assertEqual(device_map_for("cuda"), "cuda:0") + + def test_preserves_mps_model_placement(self): + self.assertIsNone(device_map_for("mps")) + + def test_places_the_model_on_the_first_xpu(self): + self.assertEqual(device_map_for("xpu"), "xpu:0") + + def test_preserves_cpu_model_placement(self): + self.assertEqual(device_map_for("cpu"), "cpu") + + +if __name__ == "__main__": + unittest.main() From 37e18a4bf3bdc7cfb54bb5ede6ab3e4ce11b6b50 Mon Sep 17 00:00:00 2001 From: Tai An Date: Fri, 4 Sep 2026 21:30:16 -0700 Subject: [PATCH 36/85] fix(quantization): record job progress without an attached listener (#11874) A quantization job that runs with no client on its progress stream stays "queued" forever, in the API and in state.json, while the finished artifact sits on disk. state.json was written once by StartJob, and the only code that advanced a job afterwards lived inside the stream callback of StreamProgress, so job state depended on somebody watching it. The backend's progress stream cannot simply gain a second reader: each job owns one queue.Queue and QuantizationProgress pops from it, so two consumers split the updates rather than both seeing them. The stream has to be opened exactly once per job. StartJob now starts watchProgress on the application context (the request context is done as soon as the handler returns). That goroutine is the single reader: it applies each update to the job -- in the cross-replica store and in state.json, terminal statuses still winning over late updates -- and republishes it in-process. StreamProgress becomes a pure reader over that fan-out and no longer loads a backend or opens a stream. A client attaching to a job that has already finished, including one hydrated from disk after a restart, gets a final event built from the stored job instead of blocking. Two paths used to end a client's stream by breaking the gRPC connection and now release it explicitly: StopJob kills the backend, so it publishes the stopped event itself; and a stream that ends without a terminal update means the backend is gone, so the job is recorded as failed rather than left running forever. Signed-off-by: Tai An --- core/services/quantization/service.go | 264 +++++++++++++++++---- core/services/quantization/service_test.go | 154 ++++++++++++ 2 files changed, 369 insertions(+), 49 deletions(-) diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go index cd9cbcead..a6a6eefcb 100644 --- a/core/services/quantization/service.go +++ b/core/services/quantization/service.go @@ -42,6 +42,28 @@ type QuantizationService struct { // jobs is the cross-replica job store: an in-memory map kept consistent across // replicas via NATS, optionally read-through to PostgreSQL in distributed mode. jobs *syncstate.SyncedMap[string, *schema.QuantizationJob] + + // progressMu guards progressSubs. + // + // A backend's per-job progress stream has a single destructive consumer: the + // backend pops each update off one queue and hands it to whoever is reading. + // So the service opens that stream exactly once per job — in watchProgress, + // started by StartJob — and fans the updates out in-process to the SSE clients + // registered here. Opening a second stream per client would make the two + // readers race for the same updates. + progressMu sync.Mutex + progressSubs map[string][]chan *schema.QuantizationProgressEvent +} + +// progressSubBuffer is the per-subscriber event buffer. It absorbs a client that +// is briefly slow; a client that falls further behind drops events rather than +// stalling the single reader of the backend stream. +const progressSubBuffer = 64 + +// isTerminalStatus reports whether a job status is final, i.e. no further +// progress update will follow. +func isTerminalStatus(status string) bool { + return status == "stopped" || status == "completed" || status == "failed" } // NewQuantizationService creates a new QuantizationService. In distributed mode @@ -59,6 +81,7 @@ func NewQuantizationService( appConfig: appConfig, modelLoader: modelLoader, configLoader: configLoader, + progressSubs: make(map[string][]chan *schema.QuantizationProgressEvent), } // Only attach a Store interface when a concrete store exists, otherwise the @@ -240,6 +263,13 @@ func (s *QuantizationService) StartJob(ctx context.Context, userID string, req s } s.saveJobState(job) + // Consume the backend's progress stream for the lifetime of the job, not for + // the lifetime of a client's SSE connection: a job that runs with nobody + // attached must still reach "completed" in the store and in state.json. The + // request ctx is done as soon as this HTTP handler returns, so the watcher + // rides the application context instead. + go s.watchProgress(s.appConfig.Context, jobID, backendName, modelID) + return &schema.QuantizationJobResponse{ ID: jobID, Status: "queued", @@ -311,6 +341,14 @@ func (s *QuantizationService) StopJob(ctx context.Context, userID, jobID string) s.saveJobState(job) s.mu.Unlock() + // Release clients attached to the progress stream: the backend process is gone, + // so the watcher will not see a terminal update to forward. + s.publishProgress(jobID, &schema.QuantizationProgressEvent{ + JobID: jobID, + Status: "stopped", + Message: "Quantization stopped by user", + }) + return nil } @@ -377,7 +415,153 @@ func (s *QuantizationService) DeleteJob(userID, jobID string) error { return nil } -// StreamProgress opens a gRPC progress stream and calls the callback for each update. +// watchProgress is the single reader of a job's backend progress stream. It +// records every transition on the job — in the cross-replica store and in +// state.json — and republishes it to the clients attached via StreamProgress. +// +// Recording here rather than in StreamProgress is the point: the backend hands +// each update to one consumer, so while StreamProgress was that consumer a job's +// state only advanced while somebody was watching it. +func (s *QuantizationService) watchProgress(ctx context.Context, jobID, backendName, modelID string) { + backendModel, err := s.modelLoader.Load( + model.WithBackendString(backendName), + model.WithModel(backendName), + model.WithModelID(modelID), + ) + if err != nil { + xlog.Warn("Failed to load backend for quantization progress", "job_id", jobID, "error", err) + return + } + + err = backendModel.QuantizationProgress(ctx, &pb.QuantizationProgressRequest{ + JobId: jobID, + }, func(update *pb.QuantizationProgressUpdate) { + s.publishProgress(jobID, s.applyProgressUpdate(ctx, jobID, update)) + }) + if err != nil { + xlog.Warn("Quantization progress stream ended with an error", "job_id", jobID, "error", err) + } + + // On shutdown leave the job alone: loadJobsFromDisk already reports jobs that + // were running at exit as stopped. + if ctx.Err() != nil { + return + } + + // A stream that ends without a terminal update means the backend is gone and + // nothing further will arrive. Record that instead of leaving the job in a + // running state forever — which is the failure this watcher exists to prevent — + // and release any client still waiting on a terminal event. + s.mu.Lock() + j, ok := s.jobs.Get(jobID) + stale := ok && !isTerminalStatus(j.Status) + if stale { + j.Status = "failed" + if j.Message == "" { + j.Message = "Backend progress stream ended before the job reported a result" + } + if err := s.jobs.Set(ctx, j); err != nil { + xlog.Warn("Failed to persist orphaned job state", "job_id", jobID, "error", err) + } + s.saveJobState(j) + } + s.mu.Unlock() + + if stale { + s.publishProgress(jobID, &schema.QuantizationProgressEvent{ + JobID: jobID, + Status: "failed", + Message: "Backend progress stream ended before the job reported a result", + }) + } +} + +// applyProgressUpdate records a backend progress update on the job and returns +// the event to hand to subscribers. +func (s *QuantizationService) applyProgressUpdate(ctx context.Context, jobID string, update *pb.QuantizationProgressUpdate) *schema.QuantizationProgressEvent { + s.mu.Lock() + if j, ok := s.jobs.Get(jobID); ok { + // Don't let progress updates overwrite terminal states + if !isTerminalStatus(j.Status) { + j.Status = update.Status + } + if update.Message != "" { + j.Message = update.Message + } + if update.OutputFile != "" { + j.OutputFile = update.OutputFile + } + if err := s.jobs.Set(ctx, j); err != nil { + xlog.Warn("Failed to persist progress update", "job_id", jobID, "error", err) + } + s.saveJobState(j) + } + s.mu.Unlock() + + // Convert extra metrics + extraMetrics := make(map[string]float32, len(update.ExtraMetrics)) + for k, v := range update.ExtraMetrics { + extraMetrics[k] = v + } + + return &schema.QuantizationProgressEvent{ + JobID: update.JobId, + ProgressPercent: update.ProgressPercent, + Status: update.Status, + Message: update.Message, + OutputFile: update.OutputFile, + ExtraMetrics: extraMetrics, + } +} + +// subscribeProgress registers a channel to receive a job's progress events. +func (s *QuantizationService) subscribeProgress(jobID string) chan *schema.QuantizationProgressEvent { + ch := make(chan *schema.QuantizationProgressEvent, progressSubBuffer) + s.progressMu.Lock() + s.progressSubs[jobID] = append(s.progressSubs[jobID], ch) + s.progressMu.Unlock() + return ch +} + +// unsubscribeProgress removes a channel registered by subscribeProgress. The +// channel is never closed, so a publish racing with an unsubscribe cannot send +// on a closed channel. +func (s *QuantizationService) unsubscribeProgress(jobID string, ch chan *schema.QuantizationProgressEvent) { + s.progressMu.Lock() + defer s.progressMu.Unlock() + + subs := s.progressSubs[jobID] + for i, c := range subs { + if c == ch { + s.progressSubs[jobID] = append(subs[:i], subs[i+1:]...) + break + } + } + if len(s.progressSubs[jobID]) == 0 { + delete(s.progressSubs, jobID) + } +} + +// publishProgress fans an event out to a job's subscribers. +func (s *QuantizationService) publishProgress(jobID string, event *schema.QuantizationProgressEvent) { + s.progressMu.Lock() + subs := append([]chan *schema.QuantizationProgressEvent(nil), s.progressSubs[jobID]...) + s.progressMu.Unlock() + + for _, ch := range subs { + select { + case ch <- event: + default: + // A subscriber that cannot keep up must not stall the reader that is + // recording job state for everyone else. + xlog.Warn("Dropping quantization progress event for a slow subscriber", "job_id", jobID) + } + } +} + +// StreamProgress calls the callback for each progress event of a job until it +// reaches a terminal status or ctx is done. It is a pure reader: the job's own +// watcher owns the backend stream and the state transitions. func (s *QuantizationService) StreamProgress(ctx context.Context, userID, jobID string, callback func(event *schema.QuantizationProgressEvent)) error { s.mu.Lock() job, ok := s.jobs.Get(jobID) @@ -391,59 +575,41 @@ func (s *QuantizationService) StreamProgress(ctx context.Context, userID, jobID } s.mu.Unlock() - streamModelID := job.ModelID - if streamModelID == "" { - streamModelID = job.Backend + "-quantize" + ch := s.subscribeProgress(jobID) + defer s.unsubscribeProgress(jobID, ch) + + // Re-read the job after subscribing: it may have finished between the lookup + // above and the subscription, and no further event would ever arrive. Jobs + // restored from disk after a restart are terminal too, and have no watcher. + s.mu.Lock() + current, ok := s.jobs.Get(jobID) + terminal := ok && isTerminalStatus(current.Status) + var final *schema.QuantizationProgressEvent + if terminal { + final = &schema.QuantizationProgressEvent{ + JobID: current.ID, + Status: current.Status, + Message: current.Message, + OutputFile: current.OutputFile, + } } - backendModel, err := s.modelLoader.Load( - model.WithBackendString(job.Backend), - model.WithModel(job.Backend), - model.WithModelID(streamModelID), - ) - if err != nil { - return fmt.Errorf("failed to load backend: %w", err) + s.mu.Unlock() + if terminal { + callback(final) + return nil } - return backendModel.QuantizationProgress(ctx, &pb.QuantizationProgressRequest{ - JobId: jobID, - }, func(update *pb.QuantizationProgressUpdate) { - // Update job status and persist - s.mu.Lock() - if j, ok := s.jobs.Get(jobID); ok { - // Don't let progress updates overwrite terminal states - isTerminal := j.Status == "stopped" || j.Status == "completed" || j.Status == "failed" - if !isTerminal { - j.Status = update.Status + for { + select { + case <-ctx.Done(): + return ctx.Err() + case event := <-ch: + callback(event) + if isTerminalStatus(event.Status) { + return nil } - if update.Message != "" { - j.Message = update.Message - } - if update.OutputFile != "" { - j.OutputFile = update.OutputFile - } - if err := s.jobs.Set(ctx, j); err != nil { - xlog.Warn("Failed to persist progress update", "job_id", jobID, "error", err) - } - s.saveJobState(j) } - s.mu.Unlock() - - // Convert extra metrics - extraMetrics := make(map[string]float32) - for k, v := range update.ExtraMetrics { - extraMetrics[k] = v - } - - event := &schema.QuantizationProgressEvent{ - JobID: update.JobId, - ProgressPercent: update.ProgressPercent, - Status: update.Status, - Message: update.Message, - OutputFile: update.OutputFile, - ExtraMetrics: extraMetrics, - } - callback(event) - }) + } } // sanitizeQuantModelName replaces non-alphanumeric characters with hyphens and lowercases. diff --git a/core/services/quantization/service_test.go b/core/services/quantization/service_test.go index 665728614..ae862ffca 100644 --- a/core/services/quantization/service_test.go +++ b/core/services/quantization/service_test.go @@ -8,6 +8,9 @@ package quantization import ( "context" + "encoding/json" + "os" + "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -16,6 +19,7 @@ import ( "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/services/distributed" "github.com/mudler/LocalAI/core/services/testutil" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" ) // newTestService builds a standalone QuantizationService wired to the given bus. @@ -175,6 +179,156 @@ var _ = Describe("QuantizationService", func() { }) }) + Describe("progress recording", func() { + var ( + bus *testutil.FakeBus + s *QuantizationService + ) + + BeforeEach(func() { + bus = testutil.NewFakeBus() + s = newTestService(bus) + }) + + AfterEach(func() { + Expect(s.Close()).To(Succeed()) + }) + + // The reported failure: a job that ran with no SSE client attached stayed + // "queued" forever, because the only code that advanced job state lived + // inside StreamProgress' stream callback. The transition is now applied by + // the job's own watcher, so it lands with nobody watching. + It("advances job state and rewrites state.json with no subscriber attached", func() { + job := &schema.QuantizationJob{ID: "job-np", UserID: "user-1", Status: "queued", CreatedAt: "2026-09-05T10:00:00Z"} + Expect(s.jobs.Set(ctx, job)).To(Succeed()) + Expect(s.progressSubs).To(BeEmpty()) + + s.applyProgressUpdate(ctx, "job-np", &pb.QuantizationProgressUpdate{ + JobId: "job-np", + Status: "completed", + Message: "Quantization complete", + OutputFile: "/data/quantization/job-np/model-q4_k_m.gguf", + }) + + got, err := s.GetJob("user-1", "job-np") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Status).To(Equal("completed")) + Expect(got.Message).To(Equal("Quantization complete")) + Expect(got.OutputFile).To(Equal("/data/quantization/job-np/model-q4_k_m.gguf")) + + data, err := os.ReadFile(filepath.Join(s.jobDir("job-np"), "state.json")) + Expect(err).ToNot(HaveOccurred()) + var persisted schema.QuantizationJob + Expect(json.Unmarshal(data, &persisted)).To(Succeed()) + Expect(persisted.Status).To(Equal("completed")) + Expect(persisted.OutputFile).To(Equal("/data/quantization/job-np/model-q4_k_m.gguf")) + }) + + It("does not let a late update overwrite a terminal status", func() { + job := &schema.QuantizationJob{ID: "job-stopped", UserID: "user-1", Status: "stopped", CreatedAt: "2026-09-05T10:00:00Z"} + Expect(s.jobs.Set(ctx, job)).To(Succeed()) + + s.applyProgressUpdate(ctx, "job-stopped", &pb.QuantizationProgressUpdate{ + JobId: "job-stopped", + Status: "quantizing", + }) + + got, err := s.GetJob("user-1", "job-stopped") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Status).To(Equal("stopped")) + }) + + // The backend hands each update to a single consumer, so every client has + // to be served from one in-process fan-out rather than its own stream. + It("delivers one update to every attached subscriber", func() { + first := s.subscribeProgress("job-fan") + second := s.subscribeProgress("job-fan") + defer s.unsubscribeProgress("job-fan", first) + defer s.unsubscribeProgress("job-fan", second) + + s.publishProgress("job-fan", &schema.QuantizationProgressEvent{JobID: "job-fan", Status: "quantizing"}) + + Expect((<-first).Status).To(Equal("quantizing")) + Expect((<-second).Status).To(Equal("quantizing")) + }) + + It("unsubscribing removes the job's entry once the last client leaves", func() { + ch := s.subscribeProgress("job-leave") + Expect(s.progressSubs).To(HaveKey("job-leave")) + s.unsubscribeProgress("job-leave", ch) + Expect(s.progressSubs).ToNot(HaveKey("job-leave")) + }) + + // A client attaching after the job finished — including a job restored from + // disk as "stopped" after a restart, which has no watcher — must not block + // waiting for an event that will never come. + It("returns a final event immediately for a job that already finished", func() { + job := &schema.QuantizationJob{ + ID: "job-done", UserID: "user-1", Status: "completed", + Message: "Quantization complete", OutputFile: "/data/quantization/job-done/model-q4_k_m.gguf", + CreatedAt: "2026-09-05T10:00:00Z", + } + Expect(s.jobs.Set(ctx, job)).To(Succeed()) + + var seen []*schema.QuantizationProgressEvent + Expect(s.StreamProgress(ctx, "user-1", "job-done", func(e *schema.QuantizationProgressEvent) { + seen = append(seen, e) + })).To(Succeed()) + + Expect(seen).To(HaveLen(1)) + Expect(seen[0].Status).To(Equal("completed")) + Expect(seen[0].OutputFile).To(Equal("/data/quantization/job-done/model-q4_k_m.gguf")) + Expect(s.progressSubs).ToNot(HaveKey("job-done")) + }) + + // StopJob kills the backend, so the watcher will never forward a terminal + // update; without an explicit release an attached client would hang. + It("releases an attached client when the job is stopped", func() { + job := &schema.QuantizationJob{ID: "job-stop", UserID: "user-1", Status: "quantizing", CreatedAt: "2026-09-05T10:00:00Z"} + Expect(s.jobs.Set(ctx, job)).To(Succeed()) + + ch := s.subscribeProgress("job-stop") + defer s.unsubscribeProgress("job-stop", ch) + + // nil modelLoader: exercise the release without standing up a backend. + s.mu.Lock() + job.Status = "stopped" + s.mu.Unlock() + s.publishProgress("job-stop", &schema.QuantizationProgressEvent{ + JobID: "job-stop", Status: "stopped", Message: "Quantization stopped by user", + }) + + event := <-ch + Expect(event.Status).To(Equal("stopped")) + Expect(isTerminalStatus(event.Status)).To(BeTrue()) + }) + + It("streams published events to a client until a terminal status arrives", func() { + job := &schema.QuantizationJob{ID: "job-live", UserID: "user-1", Status: "queued", CreatedAt: "2026-09-05T10:00:00Z"} + Expect(s.jobs.Set(ctx, job)).To(Succeed()) + + var seen []string + done := make(chan error, 1) + go func() { + done <- s.StreamProgress(ctx, "user-1", "job-live", func(e *schema.QuantizationProgressEvent) { + seen = append(seen, e.Status) + }) + }() + + Eventually(func() bool { + s.progressMu.Lock() + defer s.progressMu.Unlock() + return len(s.progressSubs["job-live"]) == 1 + }).Should(BeTrue()) + + s.publishProgress("job-live", &schema.QuantizationProgressEvent{JobID: "job-live", Status: "quantizing"}) + s.publishProgress("job-live", &schema.QuantizationProgressEvent{JobID: "job-live", Status: "completed"}) + + Eventually(done).Should(Receive(BeNil())) + Expect(seen).To(Equal([]string{"quantizing", "completed"})) + }) + }) + Describe("compile-time adapter contract", func() { It("satisfies syncstate.Store for *distributed.QuantStore", func() { // Guards against drift between the adapter and the component interface; From 84db1e643045ec29ac5e1362ca5f9bd1bfbd174e Mon Sep 17 00:00:00 2001 From: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> Date: Sun, 30 Aug 2026 08:17:56 +0200 Subject: [PATCH 37/85] fix(backends): preserve an explicit seed of 0 in sglang and vllm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #11772 exempted Temperature from the zero-filter in both backend adapters, because proto3 has no field presence and an explicit 0 is indistinguishable from "unset". Seed has exactly the same property and is still filtered: if proto_field != "Temperature" and value in (None, 0, 0.0, [], False, ""): continue A caller pinning `"seed": 0` for a reproducible run therefore gets a random seed instead, with no error and no log line — the one case where the failure is invisible precisely because the request looked deliberate. Both adapters now share a named tuple of fields whose zero is meaningful, so the next one is added in one place rather than as a second special case. Deliberately left filtered: top_k, top_p, min_p and the penalties. Their zero is not a value a caller means — sglang disables top_k with -1, not 0, so forwarding 0 there would turn a default into an invalid argument. Verified on the sglang backend (Qwen3.5-MoE, arm64): with the temperature fix alone, two identical requests at temperature 0 are byte-identical, but pinning seed 0 has no effect until this change. Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> --- backend/python/sglang/backend.py | 10 +++++++++- backend/python/sglang/test.py | 13 ++++++++++--- backend/python/vllm/backend.py | 10 +++++++++- backend/python/vllm/test.py | 8 +++++--- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index 76d99a726..12a5c6dfb 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -90,6 +90,14 @@ except Exception: _ONE_DAY_IN_SECONDS = 60 * 60 * 24 + +# proto3 has no field presence, so an explicit 0 is indistinguishable from +# "unset" and the zero-filter below would drop it. These two fields have a +# meaningful zero a caller can actually intend: temperature 0 is greedy +# decoding, and 0 is a valid seed. Silently substituting a default for either +# turns a reproducible request into a random one. +_EXPLICIT_ZERO_FIELDS = ("Temperature", "Seed") + MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1')) @@ -323,7 +331,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): if not hasattr(request, proto_field): continue value = getattr(request, proto_field) - if proto_field != "Temperature" and value in (None, 0, 0.0, [], False, ""): + if proto_field not in _EXPLICIT_ZERO_FIELDS and value in (None, 0, 0.0, [], False, ""): continue # repeated fields come back as RepeatedScalarContainer — convert if hasattr(value, "__iter__") and not isinstance(value, (str, bytes)): diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index c50ed577f..52b2d9b1f 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -128,11 +128,14 @@ class TestSglangHelpers(unittest.TestCase): self.assertNotIn("enable_thinking", kwargs_for({})) self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False) - def test_explicit_zero_temperature_is_preserved(self): - """Temperature=0 is valid greedy decoding, not an unset value.""" + def test_explicit_zero_temperature_and_seed_are_preserved(self): + """Temperature=0 is greedy decoding and 0 is a valid seed — neither is + an unset value. A dropped seed turns a reproducible request random.""" from types import SimpleNamespace servicer = self._servicer() + import sys as _sys + _SEED_KEY_FOR_TEST = _sys.modules["backend"]._SEED_KEY request = SimpleNamespace( Temperature=0, N=0, @@ -154,8 +157,12 @@ class TestSglangHelpers(unittest.TestCase): params = servicer._build_sampling_params(request) self.assertEqual(params["temperature"], 0) - # Other protobuf-default scalar fields must remain filtered. + self.assertEqual(params[_SEED_KEY_FOR_TEST], 0) + # Other protobuf-default scalar fields must remain filtered. top_k=0 in + # particular is not a value sglang accepts (-1 disables it), so it must + # keep falling through to the engine default. self.assertNotIn("top_p", params) + self.assertNotIn("top_k", params) if __name__ == "__main__": diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 7235c8e07..af072b49b 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -60,6 +60,12 @@ except ImportError: _ONE_DAY_IN_SECONDS = 60 * 60 * 24 +# proto3 has no field presence, so an explicit 0 is indistinguishable from +# "unset". These two fields have a meaningful zero a caller can intend: +# temperature 0 is greedy decoding, and 0 is a valid seed. +_EXPLICIT_ZERO_FIELDS = ("Temperature", "Seed") + + # If MAX_WORKERS are specified in the environment use it, otherwise default to 1 MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1')) @@ -553,7 +559,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): for request_field, param_field in request_to_sampling_params.items(): if hasattr(request, request_field): value = getattr(request, request_field) - if request_field == "Temperature" or value not in (None, 0, [], False): + # See _EXPLICIT_ZERO_FIELDS: temperature 0 is greedy decoding + # and 0 is a valid seed, so neither may be filtered out. + if request_field in _EXPLICIT_ZERO_FIELDS or value not in (None, 0, [], False): setattr(sampling_params, param_field, value) return sampling_params diff --git a/backend/python/vllm/test.py b/backend/python/vllm/test.py index a0679d4ff..d2932648e 100644 --- a/backend/python/vllm/test.py +++ b/backend/python/vllm/test.py @@ -121,16 +121,18 @@ class TestBackendServicer(unittest.TestCase): finally: self.tearDown() - def test_explicit_zero_temperature_is_preserved(self): - """Temperature=0 is valid greedy decoding, not an unset value.""" + def test_explicit_zero_temperature_and_seed_are_preserved(self): + """Temperature=0 is greedy decoding and 0 is a valid seed — neither is + an unset value. A dropped seed turns a reproducible request random.""" import sys, os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from backend import BackendServicer servicer = BackendServicer() - request = backend_pb2.PredictOptions(Prompt="hello", Temperature=0) + request = backend_pb2.PredictOptions(Prompt="hello", Temperature=0, Seed=0) sampling_params = servicer._build_sampling_params(request) self.assertEqual(sampling_params.temperature, 0) + self.assertEqual(sampling_params.seed, 0) # Other protobuf-default scalar fields must remain filtered. self.assertEqual(sampling_params.top_p, 0.9) From c4fe77a3693d4281345356c8e8b919fda94c3274 Mon Sep 17 00:00:00 2001 From: pos-ei-don Date: Sat, 29 Aug 2026 19:47:41 +0200 Subject: [PATCH 38/85] fix(vllm): tell the reasoning parser whether thinking was enabled vLLM's engine-based reasoning parsers derive their initial state from the chat template kwargs. Qwen3Parser: chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} self.thinking_enabled = chat_kwargs.get("enable_thinking", True) Constructed as ReasoningParser(tokenizer) the flag defaults to True, so the parser starts in the REASONING state. A completion produced with thinking disabled contains no tags at all, and every reasoning parser shape then reports the whole answer as reasoning: - engine-based parsers classify it by initial state; - BaseThinkingReasoningParser hits its documented "may not generate start token" fallback and returns (model_output, None). Either way `content = c if c is not None else generated_text` turns that into a duplicate: a Qwen3 model answering "391" with thinking off comes back as reasoning_content="391" AND content="391". Measured against Qwen3.5-MoE on vLLM 0.28, non-streaming: before thinking on reasoning=202 content="391" thinking off reasoning="391" content="391" <- duplicated after thinking on reasoning=192 content="391" thinking off reasoning="" content="391" Forward the kwargs the prompt was rendered with, which is what vLLM's own OpenAI server does; parsers that do not accept the argument keep the plain constructor. _split_reasoning() covers the older parser shape, which has no initial state to set. It only reclassifies when the parser exposes a start/end token pair and neither the completion nor the prompt ever opened a reasoning block. Truncated reasoning (block open, end token never arrived) stays reasoning, and parsers without that token pair are left untouched. Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> --- backend/python/vllm/backend.py | 79 +++++++++++++++++++++-- backend/python/vllm/test.py | 113 +++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 4 deletions(-) diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index af072b49b..32dd5e81c 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -566,6 +566,73 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): return sampling_params + def _new_reasoning_parser(self, chat_template_kwargs): + """Build the reasoning parser, telling it whether thinking is on. + + vLLM's newer parser engines decide their *initial state* from + ``chat_template_kwargs``: ``Qwen3Parser`` reads + ``chat_template_kwargs["enable_thinking"]`` and defaults to ``True``, + starting in the REASONING state. Constructed without it, a completion + produced with thinking disabled is classified as reasoning end to end, + and the answer is reported in both ``reasoning_content`` and + ``content``. + + vLLM's own OpenAI server forwards the request's chat template kwargs + here; this backend renders the template itself, so it forwards the + same dict. Older parsers do not accept the argument — fall back to the + plain constructor for those. + """ + try: + return self.reasoning_parser_cls( + self.tokenizer, chat_template_kwargs=chat_template_kwargs or {}, + ) + except TypeError: + return self.reasoning_parser_cls(self.tokenizer) + + @staticmethod + def _split_reasoning(rp, generated_text, prompt, reasoning, content): + """Decide what the reasoning parser's output actually means. + + Covers the *older* parser shape, which has no initial state to set: + ``BaseThinkingReasoningParser.extract_reasoning`` documents its own + fallback — "For models that may not generate start token, assume the + reasoning content is always at the start." When no end token is + present it returns *everything* as reasoning and ``None`` as content, + which is right for a truncated reasoning run and wrong for a + completion that never contained reasoning at all. + + Taking ``None`` content to mean "keep the raw text" then duplicates + the answer into both fields. + + The prompt says which case it is. A template with thinking on leaves + the reasoning block open (the prompt ends with the start token); with + thinking off it closes the block in the prompt, so the completion is + plain content. Parsers that expose no token pair (the engine-based + adapters, which take the ``chat_template_kwargs`` route above) keep + the parser's verdict unchanged. + """ + start = getattr(rp, "start_token", None) + end = getattr(rp, "end_token", None) + + if end and end in generated_text: + # The parser split on the end token. Empty content here means the + # model stopped right after it, not that parsing failed. + return reasoning or "", content or "" + + if not start: + # Unknown token layout — keep the previous behaviour rather than + # guess. + return reasoning or "", content if content is not None else generated_text + + if not (start in generated_text or (prompt or "").rstrip().endswith(start)): + # No end token and the block was never open: the "reasoning starts + # at the beginning" fallback does not apply to this completion. + return "", generated_text + + # Block was open and the end token never arrived — reasoning ran out of + # budget. It is all reasoning, and there is no answer to report. + return reasoning or "", content or "" + async def _predict(self, request, context, streaming=False): # Build the sampling parameters sampling_params = self._build_sampling_params(request) @@ -580,6 +647,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # Extract image paths and process images prompt = request.Prompt + # Kept in scope: the reasoning parser needs to know which chat + # template kwargs produced this prompt. + template_kwargs = {} image_paths = request.Images image_data = [self.load_image(img_path) for img_path in image_paths] @@ -590,7 +660,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # If tokenizer template is enabled and messages are provided instead of prompt, apply the tokenizer template if not request.Prompt and request.UseTokenizerTemplate and request.Messages: messages_dicts = self._messages_to_dicts(request.Messages) - template_kwargs = {"tokenize": False, "add_generation_prompt": True} + template_kwargs.update({"tokenize": False, "add_generation_prompt": True}) # Pass tools for tool calling if request.Tools: @@ -765,10 +835,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): if self.reasoning_parser_cls: try: - rp = self.reasoning_parser_cls(self.tokenizer) + rp = self._new_reasoning_parser(template_kwargs) r, c = rp.extract_reasoning(generated_text, request=None) - reasoning_content = r or "" - content = c if c is not None else generated_text + reasoning_content, content = self._split_reasoning( + rp, generated_text, prompt, r, c, + ) except Exception as e: print(f"Reasoning parser error: {e}", file=sys.stderr) diff --git a/backend/python/vllm/test.py b/backend/python/vllm/test.py index d2932648e..27a846f09 100644 --- a/backend/python/vllm/test.py +++ b/backend/python/vllm/test.py @@ -551,3 +551,116 @@ class TestStreamingToolParser(unittest.TestCase): intermediate, ["Hello ", "world", "!"], f"plain streaming changed; got {intermediate!r}", ) + + +class TestReasoningSplit(unittest.TestCase): + """Server-less tests for BackendServicer._split_reasoning. + + vLLM's BaseThinkingReasoningParser returns the whole completion as + reasoning and None as content whenever the end token is missing. Taken + literally that duplicates a thinking-disabled answer into both fields. + """ + + class _Parser: + start_token = "" + end_token = "" + + def _split(self, generated, prompt, reasoning, content): + import sys, os + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backend import BackendServicer + return BackendServicer._split_reasoning( + self._Parser(), generated, prompt, reasoning, content, + ) + + def test_thinking_off_is_not_duplicated_into_reasoning(self): + """No tags anywhere: the answer is content, and only content.""" + r, c = self._split( + "391", "user: 17*23?\n\n\n\n\n", + reasoning="391", content=None, + ) + self.assertEqual(r, "") + self.assertEqual(c, "391") + + def test_prefilled_start_tag_keeps_truncated_reasoning(self): + """Prompt left the block open and the end token never arrived + (budget exhausted): that really is all reasoning.""" + r, c = self._split( + "thinking and thinking", "user: hi\n\n", + reasoning="thinking and thinking", content=None, + ) + self.assertEqual(r, "thinking and thinking") + self.assertEqual(c, "") + + def test_end_token_present_keeps_parser_split(self): + r, c = self._split( + "adding two and two4", "user: hi\n\n", + reasoning="adding two and two", content="4", + ) + self.assertEqual(r, "adding two and two") + self.assertEqual(c, "4") + + def test_stop_right_after_end_token_yields_empty_content(self): + """Content must not fall back to the raw text — that would put the + reasoning into the answer.""" + r, c = self._split( + "reasoned", "user: hi\n\n", + reasoning="reasoned", content=None, + ) + self.assertEqual(r, "reasoned") + self.assertEqual(c, "") + + def test_unknown_token_layout_keeps_previous_behaviour(self): + class _Bare: + pass + import sys, os + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backend import BackendServicer + r, c = BackendServicer._split_reasoning( + _Bare(), "raw", "prompt", "raw", None, + ) + self.assertEqual(r, "raw") + self.assertEqual(c, "raw") + + +class TestReasoningParserConstruction(unittest.TestCase): + """The parser must learn whether thinking was on for this request. + + vLLM's engine-based parsers (Qwen3Parser and friends) read + chat_template_kwargs["enable_thinking"] and default to True, so a parser + built without it treats a thinking-disabled completion as pure reasoning. + """ + + def _servicer(self): + import sys, os + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backend import BackendServicer + s = BackendServicer() + s.tokenizer = object() + return s + + def test_chat_template_kwargs_are_forwarded(self): + seen = {} + + class _Parser: + def __init__(self, tokenizer, **kwargs): + seen.update(kwargs) + + s = self._servicer() + s.reasoning_parser_cls = _Parser + s._new_reasoning_parser({"enable_thinking": False}) + self.assertEqual( + seen.get("chat_template_kwargs"), {"enable_thinking": False}, + ) + + def test_parser_without_the_kwarg_still_builds(self): + """Older parsers take only the tokenizer — must not break them.""" + class _Old: + def __init__(self, tokenizer): + self.tokenizer = tokenizer + + s = self._servicer() + s.reasoning_parser_cls = _Old + self.assertIsInstance( + s._new_reasoning_parser({"enable_thinking": False}), _Old, + ) From 9319450aa6b0e6630f51e4df93772db00d870177 Mon Sep 17 00:00:00 2001 From: Tai An Date: Wed, 26 Aug 2026 09:18:19 -0700 Subject: [PATCH 39/85] fix(python-backends): re-attach media markers under use_tokenizer_template (#11621) With `template.use_tokenizer_template: true` the sglang and vllm backends render the prompt themselves via `tokenizer.apply_chat_template()`, and they hand it plain string content. A chat template only emits the model's own media tokens when the content is a list of parts, so the rendered prompt carries no `<|vision_start|><|image_pad|><|vision_end|>`. The pixels do reach the engine (`image_data` / `multi_modal_data`), but both engines locate them by scanning the prompt for that token, so they are discarded silently: HTTP 200, no warning, and the model answers as if no image had been attached. Add `attach_media_parts()` to the shared `python_utils` helper and call it in both backends: the last user turn is rebuilt as `[{"type": "image"} * n, {"type": "video"} * n, {"type": "text", ...}]` before templating, which makes the template emit the placeholders. The pixels keep travelling out of band exactly as before. Text-only requests are untouched - with no media the helper returns None and the original string-content path runs unchanged. If a template cannot iterate content parts (a text-only model), the parts render is caught and the request falls back to the previous string-content prompt instead of failing. Signed-off-by: Tai An --- backend/python/common/python_utils.py | 40 +++++++++++++++ backend/python/common/python_utils_test.py | 60 +++++++++++++++++++++- backend/python/sglang/backend.py | 19 +++++++ backend/python/vllm/backend.py | 35 ++++++++++--- 4 files changed, 146 insertions(+), 8 deletions(-) diff --git a/backend/python/common/python_utils.py b/backend/python/common/python_utils.py index c89813e2c..88ec0a530 100644 --- a/backend/python/common/python_utils.py +++ b/backend/python/common/python_utils.py @@ -37,6 +37,46 @@ def parse_options(options_list): return opts +def attach_media_parts(messages_dicts, n_images=0, n_videos=0): + """Rebuild the last user message as content *parts* carrying media markers. + + Backends that let the tokenizer do the templating hand plain string content + to ``apply_chat_template``, but a chat template only emits the model's own + media tokens (``<|vision_start|><|image_pad|><|vision_end|>`` for the + Qwen-VL family, and the equivalents elsewhere) when the content is a list + of parts. Without those markers the engine's multimodal processor finds + nothing to substitute and silently discards the pixels, even though they + were forwarded correctly out of band. + + Returns a new list whose last user message has + ``[{"type": "image"} * n_images, {"type": "video"} * n_videos, text]`` as + its content, or ``None`` when there is nothing to attach - no media, no + user turn, or content that is already a list of parts - so the caller can + keep using the original string-content list. + """ + if not n_images and not n_videos: + return None + idx = next( + ( + i + for i in reversed(range(len(messages_dicts))) + if messages_dicts[i].get("role") == "user" + ), + None, + ) + if idx is None: + return None + text = messages_dicts[idx].get("content") or "" + if not isinstance(text, str): + return None + parts = [{"type": "image"}] * n_images + [{"type": "video"}] * n_videos + if text: + parts.append({"type": "text", "text": text}) + patched = list(messages_dicts) + patched[idx] = dict(patched[idx], content=parts) + return patched + + def messages_to_dicts(proto_messages): """Convert proto ``Message`` objects to dicts suitable for ``apply_chat_template``. diff --git a/backend/python/common/python_utils_test.py b/backend/python/common/python_utils_test.py index c395ce92d..d12bac9c5 100644 --- a/backend/python/common/python_utils_test.py +++ b/backend/python/common/python_utils_test.py @@ -14,7 +14,7 @@ import json import types import unittest -from python_utils import messages_to_dicts, parse_options +from python_utils import attach_media_parts, messages_to_dicts, parse_options def _msg(**fields): @@ -118,5 +118,63 @@ class TestMessagesToDicts(unittest.TestCase): self.assertNotIn("tool_calls", out[0]) +class TestAttachMediaParts(unittest.TestCase): + def test_image_marker_added_to_last_user_turn(self): + messages = [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "how high is the water?"}, + ] + out = attach_media_parts(messages, n_images=1) + self.assertEqual( + out[3]["content"], + [{"type": "image"}, {"type": "text", "text": "how high is the water?"}], + ) + # Earlier turns and the input list itself are untouched. + self.assertEqual(out[:3], messages[:3]) + self.assertEqual(messages[3]["content"], "how high is the water?") + + def test_counts_and_order_images_then_videos(self): + out = attach_media_parts( + [{"role": "user", "content": "describe"}], n_images=2, n_videos=1 + ) + self.assertEqual( + out[0]["content"], + [ + {"type": "image"}, + {"type": "image"}, + {"type": "video"}, + {"type": "text", "text": "describe"}, + ], + ) + + def test_empty_text_yields_media_only_parts(self): + out = attach_media_parts([{"role": "user", "content": ""}], n_images=1) + self.assertEqual(out[0]["content"], [{"type": "image"}]) + + def test_other_message_keys_are_preserved(self): + out = attach_media_parts( + [{"role": "user", "content": "hi", "name": "bob"}], n_images=1 + ) + self.assertEqual(out[0]["name"], "bob") + + def test_no_media_is_a_no_op(self): + self.assertIsNone(attach_media_parts([{"role": "user", "content": "hi"}])) + + def test_no_user_turn_is_a_no_op(self): + self.assertIsNone( + attach_media_parts([{"role": "system", "content": "hi"}], n_images=1) + ) + + def test_content_already_parts_is_a_no_op(self): + self.assertIsNone( + attach_media_parts( + [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + n_images=1, + ) + ) + + if __name__ == "__main__": unittest.main() diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index 12a5c6dfb..dc2630fd1 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -40,6 +40,7 @@ import grpc sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) +from python_utils import attach_media_parts from grpc_auth import get_auth_interceptors from model_utils import resolve_model_reference @@ -375,6 +376,24 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): if _thinking in ("true", "false"): template_kwargs["enable_thinking"] = (_thinking == "true") + # sglang locates the attached images/videos by scanning the rendered + # prompt for the model's own media token, so the template has to be + # given content *parts* - string content renders a prompt with no + # placeholder and the media are dropped without a word (#11621). + media_dicts = attach_media_parts( + messages_dicts, len(request.Images), len(request.Videos) + ) + if media_dicts is not None: + try: + return self.tokenizer.apply_chat_template(media_dicts, **template_kwargs) + except Exception as e: + # A text-only template cannot iterate content parts; fall + # through to the text-only prompt instead of failing. + print( + f"chat template rejected multimodal content parts: {e!r}", + file=sys.stderr, + ) + try: return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) except TypeError: diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 32dd5e81c..de9af3798 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -20,6 +20,7 @@ import backend_pb2_grpc import grpc sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) +from python_utils import attach_media_parts from grpc_auth import get_auth_interceptors from model_utils import resolve_model_reference from vllm_utils import apply_options_to_engine_args, normalize_option_key @@ -673,13 +674,33 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): if _thinking in ("true", "false"): template_kwargs["enable_thinking"] = (_thinking == "true") - try: - prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) - except TypeError: - # Some tokenizers don't support tools/enable_thinking kwargs — retry without them - prompt = self.tokenizer.apply_chat_template( - messages_dicts, tokenize=False, add_generation_prompt=True - ) + # vLLM substitutes multi_modal_data into the model's own media + # token, so the template has to be given content *parts* - string + # content renders a prompt with no placeholder and the media are + # dropped without a word (#11621). + prompt = None + media_dicts = attach_media_parts( + messages_dicts, len(image_data), len(video_data) + ) + if media_dicts is not None: + try: + prompt = self.tokenizer.apply_chat_template(media_dicts, **template_kwargs) + except Exception as e: + # A text-only template cannot iterate content parts; fall + # through to the text-only prompt instead of failing. + print( + f"chat template rejected multimodal content parts: {e!r}", + file=sys.stderr, + ) + + if prompt is None: + try: + prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) + except TypeError: + # Some tokenizers don't support tools/enable_thinking kwargs — retry without them + prompt = self.tokenizer.apply_chat_template( + messages_dicts, tokenize=False, add_generation_prompt=True + ) # Generate text using the LLM engine request_id = random_uuid() From 4894056380555c32922b4aed2958302f3056a1fb Mon Sep 17 00:00:00 2001 From: pos-ei-don Date: Sat, 29 Aug 2026 18:53:54 +0200 Subject: [PATCH 40/85] fix(sglang): force reasoning when the template prefills the think tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qwen3-style chat templates append the opening tag to the *prompt* when thinking is enabled. The model therefore never generates it and emits only the reasoning text plus the closing . sglang's ReasoningParser keys off the opening tag: in_reasoning = self._in_reasoning or self.think_start_token in text if not in_reasoning: return StreamingParseResult(normal_text=text) so with such a template the entire completion — reasoning and answer, the raw in between — is returned as content and reasoning_content stays empty, no matter how reasoning_parser is configured. sglang's own OpenAI server handles this via force_reasoning = (self.template_manager.force_reasoning or self._get_reasoning_from_request(request)) This backend has no template manager, so derive the same signal from the rendered prompt: if it ends with the detector's think_start_token, the tag was prefilled and the parser is constructed with force_reasoning=True. Structured decoding is the exception, and it matters: a grammar applies from the first token, so the model cannot emit the closing tag even though the template opened the block. The whole completion is schema output and belongs in content — forcing there files it as reasoning and returns an empty answer. Measured against a JSON-schema code audit: 10107 characters of "reasoning", zero content. sglang's own server keeps the two apart for the same reason; its grammar backend owns the reasoning prefix when a reasoning parser is configured. force_reasoning is only passed when it is meant to be True, so detector defaults (DeepSeek-R1 already defaults to True) are untouched, and a prompt without a prefilled tag behaves exactly as before — which matters, because forcing unconditionally makes an answer generated with thinking off disappear into reasoning_content. The construction is factored into _new_reasoning_parser() so the streaming and non-streaming paths, which previously built the parser separately, cannot drift apart. Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> --- backend/python/sglang/backend.py | 86 ++++++++++++++++++++++++-------- backend/python/sglang/test.py | 52 +++++++++++++++++++ 2 files changed, 118 insertions(+), 20 deletions(-) diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index dc2630fd1..9c6848dc9 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -401,10 +401,67 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): messages_dicts, tokenize=False, add_generation_prompt=True, ) - def _make_parsers(self, request): + def _new_reasoning_parser(self, stream_reasoning: bool, prompt: str = "", + grammar_constrained: bool = False): + """Build a ReasoningParser for one request, or None. + + Reasoning templates come in two flavours. Some let the model emit the + opening tag, others put it into the *prompt* — Qwen3's template appends + ```` when thinking is on, so the completion starts straight in + the reasoning block and only the closing ```` ever shows up. + sglang's detector keys off the opening tag, so in that second case it + classifies the whole completion as normal content and + ``reasoning_content`` stays empty. + + sglang's own OpenAI server covers this with + ``template_manager.force_reasoning``; this backend has no template + manager, so it derives the same signal from the rendered prompt. + ``force_reasoning`` is only passed when we mean True, leaving detector + defaults (e.g. DeepSeek-R1's built-in True) untouched. + + ``grammar_constrained`` suppresses the prefill heuristic. A structured + decoding constraint applies from the first token, so the model cannot + emit the closing tag even though the template opened the block: the + whole completion is schema output and belongs in ``content``. Forcing + there files the answer as reasoning and leaves content empty. sglang's + own server keeps the two apart for the same reason — its grammar + backend owns the reasoning prefix when a reasoning parser is set. + """ + if grammar_constrained: + prompt = "" + + if not (HAS_REASONING_PARSERS and self.reasoning_parser_name): + return None + + kwargs = { + "model_type": self.reasoning_parser_name, + "stream_reasoning": stream_reasoning, + } + try: + parser = ReasoningParser(**kwargs) + except Exception as e: + print(f"ReasoningParser init failed: {e!r}", file=sys.stderr) + return None + + start = getattr(getattr(parser, "detector", None), "think_start_token", None) + if start and prompt and prompt.rstrip().endswith(start): + try: + parser = ReasoningParser(force_reasoning=True, **kwargs) + except TypeError: + # sglang without the force_reasoning kwarg: keep the default + # parser rather than failing the request. + pass + except Exception as e: + print( + f"ReasoningParser(force_reasoning=True) failed: {e!r}", + file=sys.stderr, + ) + + return parser + + def _make_parsers(self, request, prompt: str = ""): """Construct fresh per-request parser instances (stateful).""" tool_parser = None - reasoning_parser = None if HAS_TOOL_PARSERS and self.tool_parser_name and request.Tools: try: @@ -416,14 +473,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): except Exception as e: print(f"FunctionCallParser init failed: {e!r}", file=sys.stderr) - if HAS_REASONING_PARSERS and self.reasoning_parser_name: - try: - reasoning_parser = ReasoningParser( - model_type=self.reasoning_parser_name, - stream_reasoning=True, - ) - except Exception as e: - print(f"ReasoningParser init failed: {e!r}", file=sys.stderr) + reasoning_parser = self._new_reasoning_parser( + True, prompt, bool(getattr(request, "Grammar", "")), + ) return tool_parser, reasoning_parser @@ -431,7 +483,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): sampling_params = self._build_sampling_params(request) prompt = self._build_prompt(request) - tool_parser, reasoning_parser = self._make_parsers(request) + tool_parser, reasoning_parser = self._make_parsers(request, prompt) image_data = list(request.Images) if request.Images else None video_data = list(request.Videos) if request.Videos else None @@ -527,15 +579,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): final_tool_calls: List[backend_pb2.ToolCallDelta] = [] if not streaming: - final_reasoning_parser = None - if HAS_REASONING_PARSERS and self.reasoning_parser_name: - try: - final_reasoning_parser = ReasoningParser( - model_type=self.reasoning_parser_name, - stream_reasoning=False, - ) - except Exception: - final_reasoning_parser = None + final_reasoning_parser = self._new_reasoning_parser( + False, prompt, bool(getattr(request, "Grammar", "")), + ) if final_reasoning_parser is not None: try: diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index 52b2d9b1f..4cd559b09 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -128,6 +128,58 @@ class TestSglangHelpers(unittest.TestCase): self.assertNotIn("enable_thinking", kwargs_for({})) self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False) + def test_reasoning_parser_forced_when_template_prefills_think_tag(self): + """Qwen3's template puts ```` in the prompt, so the completion + never contains it. Without force_reasoning the detector treats the whole + completion as normal text and reasoning_content stays empty.""" + servicer = self._servicer() + servicer.reasoning_parser_name = "qwen3" + + # What the model actually emits when the prompt ends in "". + completion = "adding two and two4" + + forced = servicer._new_reasoning_parser(False, prompt="user: hi\n\n") + reasoning, content = forced.parse_non_stream(completion) + self.assertEqual(reasoning, "adding two and two") + self.assertEqual(content, "4") + + # No prefilled tag in the prompt: detector default, unchanged behaviour. + unforced = servicer._new_reasoning_parser(False, prompt="user: hi\n") + reasoning, content = unforced.parse_non_stream(completion) + self.assertFalse(reasoning) + self.assertEqual(content, completion) + + def test_reasoning_parser_not_forced_when_thinking_is_off(self): + """Thinking off means no ```` in the prompt either, so the answer + must not be swallowed into reasoning_content.""" + servicer = self._servicer() + servicer.reasoning_parser_name = "qwen3" + + parser = servicer._new_reasoning_parser(False, prompt="user: primes?\n") + reasoning, content = parser.parse_non_stream("2,3,5,7,11") + self.assertFalse(reasoning) + self.assertEqual(content, "2,3,5,7,11") + + def test_grammar_constrained_output_is_not_forced_into_reasoning(self): + """Structured decoding applies from the first token, so the model cannot + emit the closing tag even though the template opened the block. The whole + completion is schema output and must stay in content.""" + servicer = self._servicer() + servicer.reasoning_parser_name = "qwen3" + + schema_out = '{"findings": [{"line": 42, "issue": "off-by-one"}]}' + parser = servicer._new_reasoning_parser( + False, prompt="audit this\n\n", grammar_constrained=True, + ) + reasoning, content = parser.parse_non_stream(schema_out) + self.assertFalse(reasoning) + self.assertEqual(content, schema_out) + + def test_reasoning_parser_absent_without_configured_parser(self): + servicer = self._servicer() + servicer.reasoning_parser_name = None + self.assertIsNone(servicer._new_reasoning_parser(False, prompt="")) + def test_explicit_zero_temperature_and_seed_are_preserved(self): """Temperature=0 is greedy decoding and 0 is a valid seed — neither is an unset value. A dropped seed turns a reproducible request random.""" From 1479aef8cd213d924471615925e73360a28404e6 Mon Sep 17 00:00:00 2001 From: Anupam Mediratta Date: Sat, 5 Sep 2026 22:04:02 +0000 Subject: [PATCH 41/85] fix(deps): upgrade path-to-regexp to 8.4.0 (CVE-2026-4926) Squashed merge of #11633. Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/bun.lock | 3 ++- core/http/react-ui/package.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/http/react-ui/bun.lock b/core/http/react-ui/bun.lock index f0b034a69..4d9ce9572 100644 --- a/core/http/react-ui/bun.lock +++ b/core/http/react-ui/bun.lock @@ -53,6 +53,7 @@ "overrides": { "hono": "4.12.34", "ip-address": "10.3.1", + "path-to-regexp": "^8.4.0", }, "packages": { "@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="], @@ -807,7 +808,7 @@ "path-scurry": ["path-scurry@2.0.2", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg=="], - "path-to-regexp": ["path-to-regexp@8.3.0", "", {}, "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA=="], + "path-to-regexp": ["path-to-regexp@8.4.0", "", {}, "sha512-PuseHIvAnz3bjrM2rGJtSgo1zjgxapTLZ7x2pjhzWwlp4SJQgK3f3iZIQwkpEnBaKz6seKBADpM4B4ySkuYypg=="], "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], diff --git a/core/http/react-ui/package.json b/core/http/react-ui/package.json index 9970ba108..6d3861a79 100644 --- a/core/http/react-ui/package.json +++ b/core/http/react-ui/package.json @@ -20,7 +20,8 @@ }, "overrides": { "hono": "4.12.34", - "ip-address": "10.3.1" + "ip-address": "10.3.1", + "path-to-regexp": "^8.4.0" }, "dependencies": { "@codemirror/autocomplete": "^6.18.6", From 718357219be011873098b7377e175a7bdb85e36b Mon Sep 17 00:00:00 2001 From: pengmin Date: Sat, 5 Sep 2026 22:04:02 +0000 Subject: [PATCH 42/85] fix(ui): send collection intervals as numbers Squashed merge of #11819. Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/collections.spec.js | 29 +++++++++++++++++++ .../react-ui/src/pages/CollectionDetails.jsx | 6 ++-- core/http/react-ui/src/utils/api.js | 5 +++- docs/content/features/agents.md | 3 ++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/core/http/react-ui/e2e/collections.spec.js b/core/http/react-ui/e2e/collections.spec.js index 4fa4168dd..f3f4bd8c2 100644 --- a/core/http/react-ui/e2e/collections.spec.js +++ b/core/http/react-ui/e2e/collections.spec.js @@ -19,4 +19,33 @@ test.describe('Collections page', () => { await input.fill('my-kb') await expect(input).toHaveValue('my-kb') }) + + test('posts the source update interval as a JSON number', async ({ page }) => { + const collectionName = 'interval-regression' + const collectionPath = encodeURIComponent(collectionName) + let postedBody + + await page.route(`**/api/agents/collections/${collectionPath}/entries`, route => + route.fulfill({ contentType: 'application/json', body: JSON.stringify({ entries: [] }) })) + await page.route(`**/api/agents/collections/${collectionPath}/sources`, async route => { + if (route.request().method() === 'POST') { + postedBody = route.request().postDataJSON() + await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ status: 'ok' }) }) + } else { + await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ sources: [] }) }) + } + }) + + await page.goto(`/app/collections/${collectionPath}`) + await page.getByRole('button', { name: 'Sources' }).click() + await page.locator('#source-url').fill('https://example.com/feed') + await page.locator('#source-interval').fill('3600') + await page.getByRole('button', { name: 'Add Source' }).click() + + await expect.poll(() => postedBody).toEqual({ + url: 'https://example.com/feed', + update_interval: 3600, + }) + expect(typeof postedBody.update_interval).toBe('number') + }) }) diff --git a/core/http/react-ui/src/pages/CollectionDetails.jsx b/core/http/react-ui/src/pages/CollectionDetails.jsx index 0469df078..2ce4b6991 100644 --- a/core/http/react-ui/src/pages/CollectionDetails.jsx +++ b/core/http/react-ui/src/pages/CollectionDetails.jsx @@ -432,10 +432,12 @@ export default function CollectionDetails() { setNewSourceInterval(e.target.value)} - placeholder="e.g. 1h, 30m" + placeholder="e.g. 60 (minutes)" />