From 964be3bceb4c15f09505a82abdb1695692fb4660 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:54:29 +0200 Subject: [PATCH 001/157] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `0ed847d3140baead542abe3e5e6fe841013e7340` (#11708) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index 847ddbb20..f3ff79a7f 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=8337e4cd3861406fc04e0854b1409cd1b027fbc9 +IK_LLAMA_VERSION?=0ed847d3140baead542abe3e5e6fe841013e7340 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From a760a7ab4bd16e3e6c41ba8f543a4c9188a2760e Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:52:57 +0800 Subject: [PATCH 002/157] fix(backends): honor enable_thinking=false in sglang and vllm (#11715) Those backends only forwarded the flag when it was "true", so "false" never reached apply_chat_template and Qwen3 kept thinking on. Signed-off-by: lei_lei <96427312+leilei3167@users.noreply.github.com> --- backend/python/sglang/backend.py | 5 +++-- backend/python/sglang/test.py | 32 ++++++++++++++++++++++++++++++++ backend/python/vllm/backend.py | 6 +++--- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index c28b59a0e..ad6c6ca10 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -363,8 +363,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): template_kwargs["tools"] = json.loads(request.Tools) except json.JSONDecodeError: pass - if request.Metadata.get("enable_thinking", "").lower() == "true": - template_kwargs["enable_thinking"] = True + _thinking = request.Metadata.get("enable_thinking", "").lower() + if _thinking in ("true", "false"): + template_kwargs["enable_thinking"] = (_thinking == "true") try: return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index 92688f444..deb615883 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -96,6 +96,38 @@ class TestSglangHelpers(unittest.TestCase): servicer._apply_engine_args({}, "[1,2,3]") self.assertIn("must be a JSON object", str(ctx.exception)) + def test_build_prompt_forwards_enable_thinking(self): + from types import SimpleNamespace + + class Tok: + def __init__(self): + self.kwargs = None + + def apply_chat_template(self, messages, **kwargs): + self.kwargs = kwargs + return "PROMPT" + + def kwargs_for(metadata): + servicer = self._servicer() + tok = Tok() + servicer.tokenizer = tok + msg = SimpleNamespace( + role="user", content="hi", name="", + tool_call_id="", reasoning_content="", tool_calls="", + ) + req = SimpleNamespace( + Prompt="", UseTokenizerTemplate=True, + Messages=[msg], Tools="", Metadata=metadata, + ) + self.assertEqual(servicer._build_prompt(req), "PROMPT") + return tok.kwargs + + self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True) + # "false" used to be dropped, so Qwen3 kept thinking on + self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False) + self.assertNotIn("enable_thinking", kwargs_for({})) + self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False) + if __name__ == "__main__": unittest.main() diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 8fd3c2dc1..f3f01ec45 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -587,9 +587,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): except json.JSONDecodeError: pass - # Enable thinking mode if requested - if request.Metadata.get("enable_thinking", "").lower() == "true": - template_kwargs["enable_thinking"] = True + _thinking = request.Metadata.get("enable_thinking", "").lower() + if _thinking in ("true", "false"): + template_kwargs["enable_thinking"] = (_thinking == "true") try: prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) From f7c55788c7fdab3ba6de234fa3f8788f91bcd4ed Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 25 Aug 2026 12:54:00 +0200 Subject: [PATCH 003/157] feat(gallery): add Ornith 1.5 35B variants (#11714) Add the official Q4_K_M and Q8_0 GGUF builds with their shared BF16 vision projector. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 97 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 1c67d799b..35392e405 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -597,6 +597,103 @@ - filename: llama-cpp/mmproj/ornith-1.5-9b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/mmproj-Ornith-1.5-9B-BF16.gguf sha256: 626f9f90627402a6bf4a999111d0fbd69b5fcca7aa8ba089d69e5f10e8858e1d +- &ornith-1-5-35b-a3b + name: "ornith-1.5-35b-a3b-q4" + variants: + - model: ornith-1.5-35b-a3b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B-GGUF + description: | + Ornith-1.5-35B-A3B is an MIT-licensed Qwen3.5 mixture-of-experts model + from Ornith AI for agentic coding, reasoning, repository-level software + tasks, and tool use. It activates about 3B parameters per token and + supports text and image input with a context window of 262K tokens. + + This default entry uses the Q4_K_M GGUF and BF16 vision projector. A + higher-quality Q8_0 model is available as a variant. + license: "mit" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q4_K_M.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q4_K_M.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/Ornith-1.5-35B-Q4_K_M.gguf + sha256: 42739874cc2ccfdb8523b23fbe52e29b2a7555c8176737ca9ca0b5d59859d41f + - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf + sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d +- !!merge <<: *ornith-1-5-35b-a3b + name: "ornith-1.5-35b-a3b-q8" + variants: [] + description: | + Ornith-1.5-35B-A3B in the higher-quality Q8_0 GGUF format, with the shared + BF16 vision projector for multimodal prompts. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q8_0.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q8_0.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/Ornith-1.5-35B-Q8_0.gguf + sha256: de46c4baf4b4dd85ea438bb0f757f21c38841a353506579979bba114311658c3 + - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf + sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d - &qwen3-8-27b-obliterated name: "qwen3.8-27b-obliterated-q4" variants: From ccb9a0a088fd5630ffcd94d88c5ec7321728e7ed Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:55:43 +0200 Subject: [PATCH 004/157] chore: :arrow_up: Update 0xShug0/audio.cpp to `d25ffac094a9d5a240940b4955ea79ad9b7b4c78` (#11710) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index bee144fbc..2462a4f40 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=288a2712316470847a730e55db9ac9e5062a2b03 +AUDIO_CPP_VERSION?=d25ffac094a9d5a240940b4955ea79ad9b7b4c78 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From fa9ffc181ce2b0e7682e9b600061cb603335b3d0 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:57:12 +0200 Subject: [PATCH 005/157] chore: :arrow_up: Update ggml-org/llama.cpp to `f280b26983ad0fdb705a0d9ebf0503e76f2899b0` (#11646) * :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(llama-cpp): adapt to the common JSON API The llama.cpp bump replaces its nlohmann JSON alias with common_json. Update the gRPC adapter for the new exception, iterator, conversion, and container APIs. Assisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(turboquant): adapt the JSON exception type The shared gRPC source now follows the upstream common_json API. The TurboQuant fork still exposes nlohmann JSON and cannot compile the new exception type. Translate that exception in the fork-specific source patch so both llama.cpp variants compile from the shared adapter. Assisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(bonsai): adapt the JSON exception type The shared gRPC source uses upstream's common_json wrapper. The Bonsai fork still exposes nlohmann JSON and cannot compile that exception type.\n\nTranslate the exception in the fork-specific preparation step and verify that repeated preparation stays idempotent.\n\nAssisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(llama-cpp): let prepare register gRPC The score patch duplicated the gRPC CMake registration that prepare.sh already owns. Its stale context rejects the current upstream tools file on Darwin before compilation starts. Assisted-by: Codex:gpt-5 --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- backend/cpp/bonsai/Makefile | 2 + backend/cpp/bonsai/patch-grpc-server.sh | 24 ++++++++ backend/cpp/llama-cpp/Makefile | 2 +- backend/cpp/llama-cpp/grpc-server.cpp | 55 +++++++++---------- backend/cpp/llama-cpp/message_content.h | 7 ++- .../0001-add-server-task-type-score.patch | 12 +--- backend/cpp/turboquant/patch-grpc-server.sh | 14 +++++ scripts/build/bonsai-json-compat_test.sh | 25 +++++++++ 8 files changed, 97 insertions(+), 44 deletions(-) create mode 100644 backend/cpp/bonsai/patch-grpc-server.sh create mode 100644 scripts/build/bonsai-json-compat_test.sh diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index 96171e6a9..a3cdf980f 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -41,6 +41,7 @@ define bonsai-build # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge + bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:$(1)$(RESET)) @@ -79,6 +80,7 @@ bonsai-cpu-all: # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge + bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET)) diff --git a/backend/cpp/bonsai/patch-grpc-server.sh b/backend/cpp/bonsai/patch-grpc-server.sh new file mode 100644 index 000000000..aa9b23a50 --- /dev/null +++ b/backend/cpp/bonsai/patch-grpc-server.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Adapt the shared llama.cpp gRPC source to the older JSON API in Bonsai. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +SRC=$1 +if [[ ! -f "$SRC" ]]; then + echo "grpc-server.cpp not found at $SRC" >&2 + exit 2 +fi + +if grep -q 'common_json_error' "$SRC"; then + echo "==> patching $SRC to use the Bonsai JSON exception type" + awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" + mv "$SRC.tmp" "$SRC" + echo "==> Bonsai JSON exception patch OK" +else + echo "==> $SRC already uses a Bonsai-compatible JSON exception type, skipping" +fi diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 41861f5f2..d9e248adf 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd +LLAMA_VERSION?=f280b26983ad0fdb705a0d9ebf0503e76f2899b0 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 171ae0483..7c1a3d320 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -294,7 +294,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const } else { SRV_WRN("[TOOLS DEBUG] parse_options: Parsed tools JSON is not an array: %s\n", tools_json.dump().c_str()); } - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { SRV_WRN("Failed to parse tools JSON from proto: %s\n", e.what()); SRV_WRN("[TOOLS DEBUG] parse_options: Tools string that failed to parse: %s\n", predict->tools().c_str()); } @@ -324,7 +324,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const SRV_DBG("[TOOLS DEBUG] Received tool_choice object from Go layer: %s\n", tool_choice_json.dump().c_str()); } SRV_INF("Extracted tool_choice from proto: %s\n", predict->toolchoice().c_str()); - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { // If parsing fails, treat as string data["tool_choice"] = predict->toolchoice(); SRV_INF("Extracted tool_choice as string: %s\n", predict->toolchoice().c_str()); @@ -353,7 +353,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const // Add to data - llama.cpp server expects it as an object (map) data["logit_bias"] = logit_bias_json; SRV_INF("Using logit_bias: %s\n", predict->logitbias().c_str()); - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { SRV_ERR("Failed to parse logit_bias JSON from proto: %s\n", e.what()); } } @@ -398,7 +398,10 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const }); } - data["stop"] = predict->stopprompts(); + data["stop"] = json::array(); + for (const auto & stop : predict->stopprompts()) { + data["stop"].push_back(stop); + } // data["n_probs"] = predict->nprobs(); //TODO: images, @@ -1795,7 +1798,7 @@ public: for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j)); for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j)); } - messages_json.push_back(llama_grpc::build_reconstructed_message(rin)); + messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump())); } // Final safety check: Ensure no message has null content (Jinja templates require strings) @@ -1988,7 +1991,7 @@ public: if (!body_json.contains("chat_template_kwargs")) { body_json["chat_template_kwargs"] = json::object(); } - for (auto& el : ctk.items()) { + for (auto el : ctk.items()) { body_json["chat_template_kwargs"][el.key()] = el.value(); } } @@ -2074,30 +2077,27 @@ public: // If not using chat templates, extract files from image_data/audio_data fields // (If using chat templates, files were already extracted by oaicompat_chat_params_parse) if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) + if (data.contains("image_data") && data.at("image_data").is_array()) { - for (const auto &img : *images_data) + for (const auto &img : data.at("image_data")) { auto decoded_data = base64_decode(img["data"].get()); files.push_back(decoded_data); } } - const auto &audio_data = data.find("audio_data"); - if (audio_data != data.end() && audio_data->is_array()) + if (data.contains("audio_data") && data.at("audio_data").is_array()) { - for (const auto &audio : *audio_data) + for (const auto &audio : data.at("audio_data")) { auto decoded_data = base64_decode(audio["data"].get()); files.push_back(decoded_data); } } - const auto &video_data = data.find("video_data"); - if (video_data != data.end() && video_data->is_array()) + if (data.contains("video_data") && data.at("video_data").is_array()) { - for (const auto &video : *video_data) + for (const auto &video : data.at("video_data")) { auto decoded_data = base64_decode(video["data"].get()); files.push_back(decoded_data); @@ -2370,7 +2370,7 @@ public: for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j)); for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j)); } - messages_json.push_back(llama_grpc::build_reconstructed_message(rin)); + messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump())); } // Final safety check: Ensure no message has null content (Jinja templates require strings) @@ -2563,7 +2563,7 @@ public: if (!body_json.contains("chat_template_kwargs")) { body_json["chat_template_kwargs"] = json::object(); } - for (auto& el : ctk.items()) { + for (auto el : ctk.items()) { body_json["chat_template_kwargs"][el.key()] = el.value(); } } @@ -2649,11 +2649,10 @@ public: // If not using chat templates, extract files from image_data/audio_data fields // (If using chat templates, files were already extracted by oaicompat_chat_params_parse) if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) + if (data.contains("image_data") && data.at("image_data").is_array()) { - std::cout << "[PREDICT] Processing " << images_data->size() << " images" << std::endl; - for (const auto &img : *images_data) + std::cout << "[PREDICT] Processing " << data.at("image_data").size() << " images" << std::endl; + for (const auto &img : data.at("image_data")) { std::cout << "[PREDICT] Processing image" << std::endl; auto decoded_data = base64_decode(img["data"].get()); @@ -2661,20 +2660,18 @@ public: } } - const auto &audio_data = data.find("audio_data"); - if (audio_data != data.end() && audio_data->is_array()) + if (data.contains("audio_data") && data.at("audio_data").is_array()) { - for (const auto &audio : *audio_data) + for (const auto &audio : data.at("audio_data")) { auto decoded_data = base64_decode(audio["data"].get()); files.push_back(decoded_data); } } - const auto &video_data = data.find("video_data"); - if (video_data != data.end() && video_data->is_array()) + if (data.contains("video_data") && data.at("video_data").is_array()) { - for (const auto &video : *video_data) + for (const auto &video : data.at("video_data")) { auto decoded_data = base64_decode(video["data"].get()); files.push_back(decoded_data); @@ -3005,7 +3002,7 @@ public: } // Collect responses - json responses = json::array(); + std::vector responses; for (auto & res : all_results.results) { GGML_ASSERT(dynamic_cast(res.get()) != nullptr); responses.push_back(res->to_json()); @@ -3018,7 +3015,7 @@ public: // Crop results by request.top_n if specified int top_n = request->top_n(); if (top_n > 0 && top_n < static_cast(responses.size())) { - responses = json(responses.begin(), responses.begin() + top_n); + responses.resize(top_n); } // Set usage information backend::Usage* usage = rerankResult->mutable_usage(); diff --git a/backend/cpp/llama-cpp/message_content.h b/backend/cpp/llama-cpp/message_content.h index 4c7317ecd..0b70c8b56 100644 --- a/backend/cpp/llama-cpp/message_content.h +++ b/backend/cpp/llama-cpp/message_content.h @@ -52,14 +52,15 @@ inline nlohmann::ordered_json normalize_message_content(const std::string& role, // (#7528). A multimodal user message legitimately carries a typed-part array // ({type:text}, {type:image_url}, ...), which must be left intact. Shared by the // streaming and non-streaming paths so this invariant cannot drift between them. -inline void normalize_template_message(nlohmann::ordered_json& msg) { +template +inline void normalize_template_message(Json& msg) { if (!msg.contains("content")) { msg["content"] = ""; // templates expect the field to exist return; } - nlohmann::ordered_json& content = msg["content"]; + auto& content = msg["content"]; const std::string role = (msg.contains("role") && msg["role"].is_string()) - ? msg["role"].get() + ? msg["role"].template get() : std::string(); if (content.is_null()) { content = ""; // #7324: null would crash content[:N] slicing diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch index f056d47ce..253e8da5f 100644 --- a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch +++ b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch @@ -6,10 +6,9 @@ Subject: [PATCH 1/2] score-patch --- common/common.cpp | 6 +- common/common.h | 3 + - tools/CMakeLists.txt | 1 + tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++- tools/server/server-task.h | 47 +++++ - 5 files changed, 406 insertions(+), 9 deletions(-) + 4 files changed, 405 insertions(+), 9 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 2e3f14c..0cec0dc 100644 @@ -42,15 +41,6 @@ index 878534d..4001df2 100644 int32_t n_sequences = 1; // number of sequences to decode int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) int32_t n_outputs_max_per_seq = 1; // max outputs per sequence -diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt -index 780df32..1d2fe8f 100644 ---- a/tools/CMakeLists.txt -+++ b/tools/CMakeLists.txt -@@ -41,3 +41,4 @@ else() - add_subdirectory(fit-params) - add_subdirectory(results) - endif() -+add_subdirectory(grpc-server) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 3b5f6a1..d0e18e6 100644 --- a/tools/server/server-context.cpp diff --git a/backend/cpp/turboquant/patch-grpc-server.sh b/backend/cpp/turboquant/patch-grpc-server.sh index fa11897dd..8d1e35578 100755 --- a/backend/cpp/turboquant/patch-grpc-server.sh +++ b/backend/cpp/turboquant/patch-grpc-server.sh @@ -8,6 +8,8 @@ # so the grpc-server option parser skips the two references to # common_params::checkpoint_min_step (the default and the option handler). # That field does not exist in the fork yet; drop this once it does. +# 3. Use nlohmann's parse_error type in JSON catch clauses because the fork +# predates upstream's common_json_error wrapper. # # The fork used to lag upstream on the whole common_params_speculative refactor # (ggml-org/llama.cpp#22397/#22838/#22964), the model_tgt rename (#22838) and @@ -100,4 +102,16 @@ else echo "==> LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK" fi +# 3. The shared source follows current upstream and catches common_json_error. +# TurboQuant still exposes nlohmann::json directly, so its equivalent parse +# failures use json::parse_error instead. +if grep -q 'common_json_error' "$SRC"; then + echo "==> patching $SRC to use the TurboQuant JSON exception type" + awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" + mv "$SRC.tmp" "$SRC" + echo "==> TurboQuant JSON exception patch OK" +else + echo "==> $SRC already uses a TurboQuant-compatible JSON exception type, skipping" +fi + echo "==> all patches applied" diff --git a/scripts/build/bonsai-json-compat_test.sh b/scripts/build/bonsai-json-compat_test.sh new file mode 100644 index 000000000..c0cfc4f88 --- /dev/null +++ b/scripts/build/bonsai-json-compat_test.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +PATCHER="$ROOT/backend/cpp/bonsai/patch-grpc-server.sh" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +cat > "$WORK/grpc-server.cpp" <<'EOF' +try { + json::parse("{"); +} catch (const common_json_error& e) { +} +EOF + +bash "$PATCHER" "$WORK/grpc-server.cpp" +grep -q 'catch (const json::parse_error& e)' "$WORK/grpc-server.cpp" +! grep -q 'common_json_error' "$WORK/grpc-server.cpp" + +# A repeated preparation pass must not change the generated source. +cp "$WORK/grpc-server.cpp" "$WORK/once.cpp" +bash "$PATCHER" "$WORK/grpc-server.cpp" +cmp "$WORK/once.cpp" "$WORK/grpc-server.cpp" + +echo "PASS: Bonsai uses its fork-compatible JSON exception" From edabdf950104bf5949825b98e8001b55983ac31e Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 25 Aug 2026 17:36:02 +0200 Subject: [PATCH 006/157] feat(gallery): add Ornith 1.5 397B variants (#11716) * feat(gallery): add Ornith 1.5 397B variants Add the official Q4_K_M and Q8_0 GGUF builds with their shared BF16 vision projector. Assisted-by: Codex:gpt-5 * feat(gallery): resolve Ornith variant ordering\n\nKeep the 35B entries from master next to the 397B variants.\n\nAssisted-by: Codex:gpt-5 --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 35392e405..10690321f 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -694,6 +694,102 @@ - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d +- &ornith-1-5-397b + name: "ornith-1.5-397b-q4" + variants: + - model: ornith-1.5-397b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-397B + - https://huggingface.co/ornith-ai/Ornith-1.5-397B-GGUF + description: | + Ornith-1.5-397B is Ornith AI's MIT-licensed flagship mixture-of-experts + model for agentic coding, reasoning, repository-level tasks, and tool use. + It supports text and image input with a context window of 262K tokens. + + This default entry uses the Q4_K_M GGUF and BF16 vision projector. A + higher-quality Q8_0 model is available as a variant. + license: "mit" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-397b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-397b/Ornith-1.5-397B-Q4_K_M.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-397b/Ornith-1.5-397B-Q4_K_M.gguf + uri: huggingface://ornith-ai/Ornith-1.5-397B-GGUF/Ornith-1.5-397B-Q4_K_M.gguf + sha256: c7775e6fae1a47619c199c81b865df9014e5d724509a098377ce1c84744b6552 + - filename: llama-cpp/mmproj/ornith-1.5-397b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-397B-GGUF/mmproj-Ornith-1.5-397B-BF16.gguf + sha256: 9da8c035659d9782b80c2ca6bebdb3befb98a2d7c5a3ac5c001d5a9f02f76fdc +- !!merge <<: *ornith-1-5-397b + name: "ornith-1.5-397b-q8" + variants: [] + description: | + Ornith-1.5-397B in the higher-quality Q8_0 GGUF format, with the shared + BF16 vision projector for multimodal prompts. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-397b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-397b/Ornith-1.5-397B-Q8_0.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-397b/Ornith-1.5-397B-Q8_0.gguf + uri: huggingface://ornith-ai/Ornith-1.5-397B-GGUF/Ornith-1.5-397B-Q8_0.gguf + sha256: 1e033a38f099a5c125e3cf762ef93b6f3db071e18afc55dea6c265c4d0768d8e + - filename: llama-cpp/mmproj/ornith-1.5-397b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-397B-GGUF/mmproj-Ornith-1.5-397B-BF16.gguf + sha256: 9da8c035659d9782b80c2ca6bebdb3befb98a2d7c5a3ac5c001d5a9f02f76fdc - &qwen3-8-27b-obliterated name: "qwen3.8-27b-obliterated-q4" variants: From 33dafe37ab9aff35ed3f0876a4b70138c7840793 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:01:06 +0200 Subject: [PATCH 007/157] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `08b500b958a3f1102e6500e5c425e65517d6fb7e` (#11726) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index f3ff79a7f..e05a7a00f 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=0ed847d3140baead542abe3e5e6fe841013e7340 +IK_LLAMA_VERSION?=08b500b958a3f1102e6500e5c425e65517d6fb7e LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From f63f11eb869609a52fd5c58a2a80e66a8f0c6c17 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:01:19 +0200 Subject: [PATCH 008/157] chore: :arrow_up: Update 0xShug0/audio.cpp to `c79e58899bf13db4d78fd06372da23cc13f55b28` (#11722) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 2462a4f40..f2c742115 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=d25ffac094a9d5a240940b4955ea79ad9b7b4c78 +AUDIO_CPP_VERSION?=c79e58899bf13db4d78fd06372da23cc13f55b28 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 5755898e5732b227bc39239e870b4f856f3589fc Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:01:32 +0200 Subject: [PATCH 009/157] chore: :arrow_up: Update ggml-org/whisper.cpp to `978113305b2ead22249b881deafa131dc8884911` (#11711) :arrow_up: Update ggml-org/whisper.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/whisper/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/whisper/Makefile b/backend/go/whisper/Makefile index 214fba777..8b0f02d64 100644 --- a/backend/go/whisper/Makefile +++ b/backend/go/whisper/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # whisper.cpp version WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp -WHISPER_CPP_VERSION?=233fe1fc9b48a09e361d3594520838ca266537fe +WHISPER_CPP_VERSION?=978113305b2ead22249b881deafa131dc8884911 SO_TARGET?=libgowhisper.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From f28e8b24e65463178fb3da7430679454d03d180e Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:57:36 +0800 Subject: [PATCH 010/157] fix(ollama): accept :latest tag on model lookup (#11732) /api/tags appends :latest to untagged names, but chat and the other model endpoints looked the tagged name up as-is and 404'd. Signed-off-by: lei_lei <96427312+leilei3167@users.noreply.github.com> --- core/http/endpoints/ollama/models_test.go | 14 ++++++++++++++ core/http/middleware/request.go | 6 ++++++ core/http/middleware/request_test.go | 14 ++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/core/http/endpoints/ollama/models_test.go b/core/http/endpoints/ollama/models_test.go index b13cf59a0..c4d0d6b5e 100644 --- a/core/http/endpoints/ollama/models_test.go +++ b/core/http/endpoints/ollama/models_test.go @@ -146,6 +146,20 @@ parameters: Expect(resp.Details.Format).To(Equal("gguf")) Expect(resp.Details.Families).ToNot(BeEmpty()) }) + + It("looks up the model when the Ollama :latest tag is included", func() { + writeConfig("chat", ` +name: chat +backend: llama-cpp +template: + chat: "{{ .Input }}" +parameters: + model: Llama-3-8B-Q4_K_M.gguf +`) + resp := callShow("chat:latest") + Expect(resp.Details.Format).To(Equal("gguf")) + Expect(resp.Capabilities).To(ContainElement("completion")) + }) }) Describe("ListModelsEndpoint", func() { diff --git a/core/http/middleware/request.go b/core/http/middleware/request.go index 1599ef05c..080a0b73c 100644 --- a/core/http/middleware/request.go +++ b/core/http/middleware/request.go @@ -141,6 +141,12 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR } modelName := input.ModelName(nil) + // Ollama-compat /api/tags appends ":latest" to untagged names. + // Strip it for lookup so the listed name works on /api/chat, + // /v1/chat/completions, and the other model-bearing endpoints. + if strings.HasSuffix(modelName, ":latest") { + modelName = strings.TrimSuffix(modelName, ":latest") + } cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(modelName, re.applicationConfig) if err != nil { diff --git a/core/http/middleware/request_test.go b/core/http/middleware/request_test.go index 1b00c7f02..afaf8d9c8 100644 --- a/core/http/middleware/request_test.go +++ b/core/http/middleware/request_test.go @@ -82,6 +82,13 @@ var _ = Describe("SetModelAndConfig middleware", func() { Expect(resp.Error.Message).To(ContainSubstring("not found")) Expect(resp.Error.Type).To(Equal("invalid_request_error")) }) + + It("still 404s when :latest is appended to an unknown model", func() { + rec := postJSON(app, "/v1/chat/completions", + `{"model":"nonexistent-model:latest","messages":[{"role":"user","content":"hi"}]}`) + + Expect(rec.Code).To(Equal(http.StatusNotFound)) + }) }) Context("when the model exists as a config file", func() { @@ -97,6 +104,13 @@ var _ = Describe("SetModelAndConfig middleware", func() { Expect(rec.Code).To(Equal(http.StatusOK)) }) + + It("accepts the Ollama :latest tag that /api/tags appends", func() { + rec := postJSON(app, "/v1/chat/completions", + `{"model":"test-model:latest","messages":[{"role":"user","content":"hi"}]}`) + + Expect(rec.Code).To(Equal(http.StatusOK)) + }) }) Context("when the model exists as a pre-loaded config", func() { From aea477932d88e6ba6de24f30e01135d0f1764799 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:57:58 +0200 Subject: [PATCH 011/157] chore(model-gallery): :arrow_up: update checksum (#11730) :arrow_up: Checksum updates in gallery/index.yaml Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 10690321f..f3a069550 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -45,8 +45,8 @@ use_tokenizer_template: true files: - filename: llama-cpp/models/Qwen3.8-27B-DFlash2-Q4_K_M/Qwen3.8-27B-DFlash2-Q4_K_M.gguf - sha256: 18a380efc9b7ed8d88677fc895f5c11ae170653434ee378f7348f715c14d0594 uri: https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf + sha256: 1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd - name: "huihui-qwen3.8-27b-abliterated" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From e7b83ef7c0014036c14de82509d4b7e671e0b5db Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:58:17 +0200 Subject: [PATCH 012/157] Fix flaky "tests-apple" CI job in modeladmin test suite (#11717) * Initial plan * tests: raise default Eventually timeout for modeladmin suite to fix flaky macOS CI Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- core/services/modeladmin/modeladmin_suite_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/services/modeladmin/modeladmin_suite_test.go b/core/services/modeladmin/modeladmin_suite_test.go index a4332e395..b0a053167 100644 --- a/core/services/modeladmin/modeladmin_suite_test.go +++ b/core/services/modeladmin/modeladmin_suite_test.go @@ -2,6 +2,7 @@ package modeladmin import ( "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -9,5 +10,12 @@ import ( func TestModelAdmin(t *testing.T) { RegisterFailHandler(Fail) + // Several specs in this suite coordinate goroutines through + // Eventually/Consistently on unbuffered-ish channels (e.g. the + // blockingRevisionLifecycle helper). Gomega's 1s default timeout can be + // too tight on slower or loaded CI runners (notably macOS runners), + // causing spurious "Timed out after 1.005s" failures even though the + // goroutines eventually make progress. Give them more headroom. + SetDefaultEventuallyTimeout(5 * time.Second) RunSpecs(t, "modeladmin test suite") } From 5c08ec0382eb644076079affd4b46e87e6c01a7d Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:58:37 +0200 Subject: [PATCH 013/157] chore: :arrow_up: Update ggml-org/llama.cpp to `eab8ee41f889ef7823af517e8098fb8a9b3cf601` (#11724) :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index d9e248adf..ae8cff292 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=f280b26983ad0fdb705a0d9ebf0503e76f2899b0 +LLAMA_VERSION?=eab8ee41f889ef7823af517e8098fb8a9b3cf601 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From 15f12074ca662708b193084fd35753bc2d38c0d4 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:59:12 +0200 Subject: [PATCH 014/157] chore: :arrow_up: Update leejet/stable-diffusion.cpp to `50d640568388f876b0d63ee6ddb6bc86d997ec64` (#11725) :arrow_up: Update leejet/stable-diffusion.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/stablediffusion-ggml/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 1c1552074..4aa9b09d0 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # stablediffusion.cpp (ggml) STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=97d2990807fe6d558e395f8764198d7c7e7b411c +STABLEDIFFUSION_GGML_VERSION?=50d640568388f876b0d63ee6ddb6bc86d997ec64 CMAKE_ARGS+=-DGGML_MAX_NAME=128 From 5dab4fcde97773f99fdf6c7c3cf7d4f37c5e45f9 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Wed, 26 Aug 2026 09:01:55 +0200 Subject: [PATCH 015/157] feat(gallery): add Granite 4.2 variants (#11719) Add the official IBM Q4_K_M and Q8_0 GGUF builds for the 3B, 8B, and 30B Granite 4.2 models. Assisted-by: Codex:gpt-5.6-sol Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 211 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index f3a069550..b1b41fe93 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,215 @@ --- +- &granite-4-2-3b + name: "granite-4.2-3b-q4" + variants: + - model: granite-4.2-3b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ibm-granite/granite-4.2-3b + - https://huggingface.co/ibm-granite/granite-4.2-3b-GGUF + description: | + IBM Granite 4.2 3B is a compact multilingual reasoning model for chat, + coding, long-context tasks, and tool use. This entry uses the Q4_K_M + GGUF; a higher-fidelity Q8_0 build is available as a variant. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - granite + - multilingual + - reasoning + - thinking + - coding + - tools + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q4_K_M.gguf + uri: huggingface://ibm-granite/granite-4.2-3b-GGUF/granite-4.2-3b-Q4_K_M.gguf + sha256: 20e436143017578687f7f848225cc6c6038126c84149192229c7dff6e4e0f427 +- !!merge <<: *granite-4-2-3b + name: "granite-4.2-3b-q8" + variants: [] + description: | + IBM Granite 4.2 3B in the higher-fidelity Q8_0 GGUF format. It is a + compact multilingual reasoning model for chat, coding, and tool use. + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q8_0.gguf + uri: huggingface://ibm-granite/granite-4.2-3b-GGUF/granite-4.2-3b-Q8_0.gguf + sha256: 9e97320b131445ab8d9098cafb48001e9925d879e71486a8af4db4c803c55394 +- &granite-4-2-8b + name: "granite-4.2-8b-q4" + variants: + - model: granite-4.2-8b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ibm-granite/granite-4.2-8b + - https://huggingface.co/ibm-granite/granite-4.2-8b-GGUF + description: | + IBM Granite 4.2 8B is a multilingual reasoning model for chat, coding, + long-context tasks, and tool use. This entry uses the Q4_K_M GGUF; a + higher-fidelity Q8_0 build is available as a variant. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - granite + - multilingual + - reasoning + - thinking + - coding + - tools + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-8b/granite-4.2-8b-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-8b/granite-4.2-8b-Q4_K_M.gguf + uri: huggingface://ibm-granite/granite-4.2-8b-GGUF/granite-4.2-8b-Q4_K_M.gguf + sha256: 16a9369d0805f80b7377d25d87f937a90c05dc04ad79173a52001e42c9aab311 +- !!merge <<: *granite-4-2-8b + name: "granite-4.2-8b-q8" + variants: [] + description: | + IBM Granite 4.2 8B in the higher-fidelity Q8_0 GGUF format. It is a + multilingual reasoning model for chat, coding, and tool use. + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-8b/granite-4.2-8b-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-8b/granite-4.2-8b-Q8_0.gguf + uri: huggingface://ibm-granite/granite-4.2-8b-GGUF/granite-4.2-8b-Q8_0.gguf + sha256: fb66ad5750680c77c76b9dc095961375f14fc88ba15fd5fc084ef6e6701bad77 +- &granite-4-2-30b + name: "granite-4.2-30b-q4" + variants: + - model: granite-4.2-30b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ibm-granite/granite-4.2-30b + - https://huggingface.co/ibm-granite/granite-4.2-30b-GGUF + description: | + IBM Granite 4.2 30B is the family's flagship multilingual reasoning model + for chat, coding, long-context tasks, and tool use. This entry uses the + Q4_K_M GGUF; a higher-fidelity Q8_0 build is available as a variant. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - granite + - multilingual + - reasoning + - thinking + - coding + - tools + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q4_K_M.gguf + uri: huggingface://ibm-granite/granite-4.2-30b-GGUF/granite-4.2-30b-Q4_K_M.gguf + sha256: f299dace85d77ee0e24ca0b6720bd211b1f4cd1f65b9c0a2111e79692e29c9be +- !!merge <<: *granite-4-2-30b + name: "granite-4.2-30b-q8" + variants: [] + description: | + IBM Granite 4.2 30B in the higher-fidelity Q8_0 GGUF format. It is the + family's flagship multilingual reasoning model for chat, coding, and tool + use. + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q8_0.gguf + uri: huggingface://ibm-granite/granite-4.2-30b-GGUF/granite-4.2-30b-Q8_0.gguf + sha256: 005b0933353e9ba219b26e2667705bdb8dbc74eb50e4a4e6cb70fca108710f81 - name: "qwen3.8-27b-dflash2" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From bbd3ab5a1458498895fa142b633579ad3436471b Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Wed, 26 Aug 2026 09:02:31 +0200 Subject: [PATCH 016/157] feat(gallery): add Tiel-Coder 35B variants (#11723) Add Q4_K_XL, MTP Q4_K_XL, and Q8_K_XL builds with their BF16 vision projectors. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 134 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index b1b41fe93..4d8fe4d31 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -905,6 +905,140 @@ - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d +- &tiel-coder-35b-a3b + name: "tiel-coder-35b-a3b-q4" + variants: + - model: tiel-coder-35b-a3b-q4-mtp + - model: tiel-coder-35b-a3b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B + - https://huggingface.co/peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF + description: | + Tiel-Coder-35B-A3B is a 35B-parameter mixture-of-experts model for coding, + reasoning, tool use, and vision tasks. This default entry uses the + Q4_K_XL GGUF and BF16 vision projector. + license: "mit" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tiel-coder-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/tiel-coder-35b-a3b/Tiel-Coder-35B-A3B-UD-Q4_K_XL.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tiel-coder-35b-a3b/Tiel-Coder-35B-A3B-UD-Q4_K_XL.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF/Tiel-Coder-35B-A3B-UD-Q4_K_XL.gguf + sha256: 9286a94c453c6a40ad51982c3dc88df4bba32fee9efad06e4588c83c059cf17c + - filename: llama-cpp/mmproj/tiel-coder-35b-a3b/mmproj-BF16.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF/mmproj-BF16.gguf + sha256: d9ce31026d1cb1f3f8d5152e2e2a014d9d2b302b6c93a7dc07bb0a0487f52837 +- !!merge <<: *tiel-coder-35b-a3b + name: "tiel-coder-35b-a3b-q4-mtp" + variants: [] + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B + - https://huggingface.co/peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP + description: | + Tiel-Coder-35B-A3B in Q4_K_XL format with MTP speculative decoding and a + BF16 vision projector. + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + - mtp + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tiel-coder-35b-a3b-mtp/mmproj-BF16.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + parameters: + model: llama-cpp/models/tiel-coder-35b-a3b-mtp/Tiel-Coder-35B-A3B-MTP-UD-Q4_K_XL.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tiel-coder-35b-a3b-mtp/Tiel-Coder-35B-A3B-MTP-UD-Q4_K_XL.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP/Tiel-Coder-35B-A3B-MTP-UD-Q4_K_XL.gguf + sha256: 10960d1d6477b08ed36a0e542e571b473022023c25b8315b0cf8c33c57e98ccd + - filename: llama-cpp/mmproj/tiel-coder-35b-a3b-mtp/mmproj-BF16.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP/mmproj-BF16.gguf + sha256: d9ce31026d1cb1f3f8d5152e2e2a014d9d2b302b6c93a7dc07bb0a0487f52837 +- !!merge <<: *tiel-coder-35b-a3b + name: "tiel-coder-35b-a3b-q8" + variants: [] + description: | + Tiel-Coder-35B-A3B in the higher-quality Q8_K_XL GGUF format, with the + BF16 vision projector for multimodal prompts. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tiel-coder-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/tiel-coder-35b-a3b/Tiel-Coder-35B-A3B-UD-Q8_K_XL.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tiel-coder-35b-a3b/Tiel-Coder-35B-A3B-UD-Q8_K_XL.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF/Tiel-Coder-35B-A3B-UD-Q8_K_XL.gguf + sha256: 883faacea54421f80f5d2713b344c69fbd2c76093a8cc4e2694dbb6958c4d699 + - filename: llama-cpp/mmproj/tiel-coder-35b-a3b/mmproj-BF16.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF/mmproj-BF16.gguf + sha256: d9ce31026d1cb1f3f8d5152e2e2a014d9d2b302b6c93a7dc07bb0a0487f52837 - &ornith-1-5-397b name: "ornith-1.5-397b-q4" variants: From fa19b08f354ac54538ede9d093be60b263e35a6f Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:16:49 +0200 Subject: [PATCH 017/157] chore: :arrow_up: Update mudler/vllm.cpp to `6738e0b4639199f3ff0998815e4d32bfa7fe5be2` (#11647) * :arrow_up: Update mudler/vllm.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(vllm-cpp): mirror ABI v23 The new engine pin reports ABI v23 and appends mmproj_path to vllm_model_params. LocalAI still declares v21, so the build-time ABI guard rejects every backend build. Grow the Go mirror by the appended pointer and update its offset checks. ABI v23 adds a video function but does not change the mirrored text structs. Assisted-by: Codex:gpt-5.6 [systematic-debugging] --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: localai-org-maint-bot --- backend/go/vllm-cpp/Makefile | 2 +- backend/go/vllm-cpp/govllmcpp.go | 5 +++-- backend/go/vllm-cpp/vllmcpp_test.go | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 754ce5560..d403319f5 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=438305e1577768ec0f75729456a4c8b9f425e2ee +VLLM_CPP_VERSION?=6738e0b4639199f3ff0998815e4d32bfa7fe5be2 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index b880105af..20e587a6b 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v21). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v23). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 21 +const abiVersion = 23 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -83,6 +83,7 @@ type cModelParams struct { LanguageModelOnly int32 // 0 = multimodal inputs enabled (ABI v19) _ [4]byte LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19) + MMProjPath uintptr // const char*; NULL = no GGUF projector (ABI v22) } // cSamplingParams mirrors vllm_sampling_params (structured fields included). diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 681ca4d4e..55f6e211a 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v21) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v23) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(21)) + Expect(abiVersion).To(Equal(23)) }) It("cModelParams matches vllm_model_params", func() { @@ -51,7 +51,8 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104))) Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112))) Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(128))) + Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(128))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136))) }) It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() { From 6f6ddba746a2d07172f21b44a9d646bef00edbad Mon Sep 17 00:00:00 2001 From: Szymon Podeszwa <2962046+sz-po@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:33:37 +0200 Subject: [PATCH 018/157] fix(deps): bump go-m1cpu to v0.2.2 to fix SIGSEGV on Apple M5 (#11736) go-m1cpu v0.1.6 runs its cgo initialiser from a package init(), where getFrequency() dereferences the CFTypeRef returned by IORegistryEntryCreateCFProperty without a NULL check. On Apple M5 the pmgr IORegistry node does not expose voltage-states5-sram / voltage-states1-sram in the shape v0.1.6 expects, so the call returns NULL and CFDataGetLength(NULL) faults before main() runs. Every command dies, including local-ai --version. The package is linked indirectly: cmd/local-ai reaches gopsutil/v3/{process,disk}, which pull in gopsutil/v3/cpu on darwin, which calls m1cpu.IsAppleSilicon() and m1cpu.PCoreHz(). v0.2.2 adds the missing NULL guard and moves the IORegistry probe out of init() behind a lazy sync.Once. The exported Go API is unchanged and the non-darwin stub is byte-identical, so gopsutil/v3 compiles against it untouched and no other platform is affected. Bumping gopsutil/v3 is not an alternative: v3.24.5 is the final v3 release, so the v3 line will never carry this fix. Fixes #11735 Assisted-by: Claude:claude-opus-5 Signed-off-by: Szymon Podeszwa <2962046+sz-po@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5418b351d..5ed7e0b51 100644 --- a/go.mod +++ b/go.mod @@ -498,7 +498,7 @@ require ( github.com/quic-go/quic-go v0.59.0 // indirect github.com/quic-go/webtransport-go v0.10.0 // indirect github.com/rivo/uniseg v0.4.7 - github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shoenig/go-m1cpu v0.2.2 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d // indirect diff --git a/go.sum b/go.sum index 6169840bd..6a0be2c19 100644 --- a/go.sum +++ b/go.sum @@ -1263,8 +1263,11 @@ github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfx github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/go-m1cpu v0.2.2 h1:4nc55oVv7nygGnfI9bhLCLzUEs4794y0Bkqx4q2zy7Y= +github.com/shoenig/go-m1cpu v0.2.2/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= From 0a89fdb1d00a1e1e0cbbde9bd60807fa93b6ce51 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:18:18 +0200 Subject: [PATCH 019/157] chore(model-gallery): :arrow_up: update checksum (#11742) :arrow_up: Checksum updates in gallery/index.yaml Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 4d8fe4d31..b44c4b427 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1860,7 +1860,7 @@ files: - filename: llama-cpp/models/nemotron-3.5-lightning-30b-a3b/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf uri: huggingface://ggml-org/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf - sha256: b0e25ce2d301930e706d549a59d18c5969dcec664fdf7466435287b03fefda36 + sha256: d4cc4c9fffaa356db8b49cfaba9233609cfb29b4dd89d827665210dc1cc8dbbb - &muse-glimmer-30b name: "muse-glimmer-30b" variants: From 74b885c31a3d62c5d3149ecd689dcbfc4130d515 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:53:12 +0200 Subject: [PATCH 020/157] chore: :arrow_up: Update PrismML-Eng/llama.cpp to `312bb2a93ea2bf798333fa859614fbf913ecb9e2` (#11740) :arrow_up: Update PrismML-Eng/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/bonsai/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index a3cdf980f..522791d42 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of the `prism` branch on https://github.com/PrismML-Eng/llama.cpp. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -BONSAI_VERSION?=9ca265a57f85f2117942490f421f64a226dd9847 +BONSAI_VERSION?=312bb2a93ea2bf798333fa859614fbf913ecb9e2 LLAMA_REPO?=https://github.com/PrismML-Eng/llama.cpp CMAKE_ARGS?= From 8712d37e2efd698af63d25c0e7554303a10c73c4 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:53:25 +0200 Subject: [PATCH 021/157] chore: :arrow_up: Update vllm-project/vllm cu130 wheel to `0.28.0` (#11741) :arrow_up: Update vllm-project/vllm cu130 wheel Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/python/vllm/requirements-cublas13-after.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/python/vllm/requirements-cublas13-after.txt b/backend/python/vllm/requirements-cublas13-after.txt index 34f4f50ff..b519b73fe 100644 --- a/backend/python/vllm/requirements-cublas13-after.txt +++ b/backend/python/vllm/requirements-cublas13-after.txt @@ -3,8 +3,8 @@ # on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match # so uv consults this index alongside PyPI. ---extra-index-url https://wheels.vllm.ai/0.27.1/cu130 +--extra-index-url https://wheels.vllm.ai/0.28.0/cu130 # VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh), # which pins this exact vLLM version. Bumping vllm here means coordinating with a # vllm-metal release that supports the new version, or macOS/Metal builds break. -vllm==0.27.1 +vllm==0.28.0 From 460c22bff65b23f1765656c21d6aa79c6a0e4194 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:53:36 +0200 Subject: [PATCH 022/157] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `ef40550042973817ac391ca95a2ff041f512257b` (#11743) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index e05a7a00f..34fab9e6b 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=08b500b958a3f1102e6500e5c425e65517d6fb7e +IK_LLAMA_VERSION?=ef40550042973817ac391ca95a2ff041f512257b LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From 1b4c4853fb86a622fd657b1713efc7d70cf30bd2 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:53:49 +0200 Subject: [PATCH 023/157] chore: :arrow_up: Update ggml-org/llama.cpp to `925e1179947ea0c0ebfb0032df18af3a729822be` (#11744) :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index ae8cff292..bf3e20aab 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=eab8ee41f889ef7823af517e8098fb8a9b3cf601 +LLAMA_VERSION?=925e1179947ea0c0ebfb0032df18af3a729822be LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From 1070cb12450328fa684de11717985fdec78e66aa Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:54:02 +0200 Subject: [PATCH 024/157] chore: :arrow_up: Update 0xShug0/audio.cpp to `db21cbdd60f3d2ff62114bc863781ff8073ac39b` (#11746) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index f2c742115..4e5391301 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=c79e58899bf13db4d78fd06372da23cc13f55b28 +AUDIO_CPP_VERSION?=db21cbdd60f3d2ff62114bc863781ff8073ac39b AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From e58dabf75f03a0b65f95d91ee06f9c8b1e16f226 Mon Sep 17 00:00:00 2001 From: "Plamen K. Kosseff" <333840+blackd@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:32:32 +0300 Subject: [PATCH 025/157] feat(ui): add 'Focus mode' option in chat settings to persistently toggle the sidebar auto-collapse (#11750) Assisted-by: Claude:claude-fable-5 Signed-off-by: Plamen K. Kosseff --- .../http/react-ui/public/locales/en/chat.json | 2 ++ core/http/react-ui/src/pages/Chat.jsx | 28 +++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/core/http/react-ui/public/locales/en/chat.json b/core/http/react-ui/public/locales/en/chat.json index f004cee56..f1c058790 100644 --- a/core/http/react-ui/public/locales/en/chat.json +++ b/core/http/react-ui/public/locales/en/chat.json @@ -33,6 +33,8 @@ "title": "Chat Settings", "manageMode": "Manage mode", "manageModeDesc": "Let this chat install models, switch backends, and edit configs by talking to LocalAI.", + "focusMode": "Focus mode", + "focusModeDesc": "Collapse the sidebar and slim the header while a conversation is active. Esc restores them temporarily; turn this off to keep the full layout.", "systemPrompt": "System Prompt", "systemPromptPlaceholder": "You are a helpful assistant...", "temperature": "Temperature", diff --git a/core/http/react-ui/src/pages/Chat.jsx b/core/http/react-ui/src/pages/Chat.jsx index 5866a2514..d799efba0 100644 --- a/core/http/react-ui/src/pages/Chat.jsx +++ b/core/http/react-ui/src/pages/Chat.jsx @@ -21,6 +21,8 @@ import { useOperations } from '../hooks/useOperations' import { relativeTime } from '../utils/format' import { copyToClipboard } from '../utils/clipboard' +const FOCUS_MODE_KEY = 'localai_chat_focus_mode' + function getLastMessagePreview(chat) { if (!chat.history || chat.history.length === 0) return '' for (let i = chat.history.length - 1; i >= 0; i--) { @@ -405,12 +407,20 @@ export default function Chat() { // Focus mode: once a conversation has at least one message we slim the // surrounding chrome (collapse the global app rail, fade non-essential // header items). Esc gives the user back the full chrome for the rest of - // this session. + // this session. The settings drawer offers a persistent opt-out. const isInConversation = (activeChat?.history?.length || 0) > 0 const [focusOverride, setFocusOverride] = useState(false) - const focusActive = isInConversation && !focusOverride + const [focusModeEnabled, setFocusModeEnabled] = useState(() => { + try { return localStorage.getItem(FOCUS_MODE_KEY) !== 'false' } catch (_) { return true } + }) + const focusActive = focusModeEnabled && isInConversation && !focusOverride const prevAppCollapseRef = useRef(null) + const toggleFocusMode = (next) => { + setFocusModeEnabled(next) + try { localStorage.setItem(FOCUS_MODE_KEY, String(next)) } catch (_) {} + } + const artifacts = useMemo( () => canvasMode ? extractCodeArtifacts(activeChat?.history, 'role', 'assistant') : [], [activeChat?.history, canvasMode] @@ -1110,6 +1120,20 @@ export default function Chat() { /> )} +
+
+ + {t('settings.focusMode')} + + + {t('settings.focusModeDesc')} + +
+ +