From 615446ae755bcc8fb19c3e0720339059ff2b9ff6 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 12 Oct 2026 09:35:15 +0200 Subject: [PATCH] chore: :arrow_up: Update ggml-org/llama.cpp to `f2cc7282ce0042641df55e0e563b75e824e2b647` (#12601) * :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(llama-cpp): preserve chat session state Newer llama.cpp parsers return template state outside the request JSON. Pass that state to both inference tasks and response readers so grammar, tool calls, and reasoning remain available in both chat paths. Detect the parser API during preparation to preserve older fork builds. Assisted-by: Codex:gpt-6 --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- backend/cpp/llama-cpp/grpc-server.cpp | 32 ++++++++++++++++++++++++ backend/cpp/llama-cpp/prepare.sh | 7 ++++++ docs/content/features/text-generation.md | 5 +++- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 2ced9a9b7..5c02e597b 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=de7fa0a3c6a2e1b4cd9f22eb8d6bf5b12dbdb63b +LLAMA_VERSION?=f2cc7282ce0042641df55e0e563b75e824e2b647 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index a3e8f0dcb..f6bc0f34d 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -1783,6 +1783,9 @@ public: std::string prompt_str; std::vector files; // Declare files early so it's accessible in both branches +#if LOCALAI_HAS_CHAT_SESSION + common_chat_session chat_session; +#endif // Handle chat templates when UseTokenizerTemplate is enabled and Messages are provided if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_params.tmpls != nullptr) { // Convert proto Messages to JSON format compatible with oaicompat_chat_params_parse @@ -2038,7 +2041,12 @@ public: } } +#if LOCALAI_HAS_CHAT_SESSION + json parsed_data = oaicompat_chat_params_parse( + ctx_server.impl->vocab, body_json, ctx_server.impl->chat_params, files, chat_session); +#else json parsed_data = oaicompat_chat_params_parse(body_json, ctx_server.impl->chat_params, files); +#endif // Debug: Log tools after template processing if (parsed_data.contains("tools")) { @@ -2157,6 +2165,10 @@ public: ctx_server.get_meta().slot_n_ctx, ctx_server.get_meta().logit_bias_eog, data); +#endif +#if LOCALAI_HAS_CHAT_SESSION + // The parser now keeps grammar, stops and message spans in the session. + task.apply_chat_session(chat_session); #endif task.id_slot = json_value(data, "id_slot", -1); @@ -2171,7 +2183,11 @@ public: tasks.push_back(std::move(task)); } +#if LOCALAI_HAS_CHAT_SESSION + rd.post_tasks(std::move(tasks), chat_session); +#else rd.post_tasks(std::move(tasks)); +#endif } catch (const std::exception & e) { return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, e.what()); } @@ -2368,6 +2384,9 @@ public: std::string prompt_str; std::vector files; // Declare files early so it's accessible in both branches +#if LOCALAI_HAS_CHAT_SESSION + common_chat_session chat_session; +#endif // Handle chat templates when UseTokenizerTemplate is enabled and Messages are provided if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_params.tmpls != nullptr) { // Convert proto Messages to JSON format compatible with oaicompat_chat_params_parse @@ -2624,7 +2643,12 @@ public: } } +#if LOCALAI_HAS_CHAT_SESSION + json parsed_data = oaicompat_chat_params_parse( + ctx_server.impl->vocab, body_json, ctx_server.impl->chat_params, files, chat_session); +#else json parsed_data = oaicompat_chat_params_parse(body_json, ctx_server.impl->chat_params, files); +#endif // Debug: Log tools after template processing if (parsed_data.contains("tools")) { @@ -2746,6 +2770,10 @@ public: ctx_server.get_meta().slot_n_ctx, ctx_server.get_meta().logit_bias_eog, data); +#endif +#if LOCALAI_HAS_CHAT_SESSION + // The parser now keeps grammar, stops and message spans in the session. + task.apply_chat_session(chat_session); #endif task.id_slot = json_value(data, "id_slot", -1); @@ -2758,7 +2786,11 @@ public: tasks.push_back(std::move(task)); } +#if LOCALAI_HAS_CHAT_SESSION + rd.post_tasks(std::move(tasks), chat_session); +#else rd.post_tasks(std::move(tasks)); +#endif } catch (const std::exception & e) { return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, e.what()); } diff --git a/backend/cpp/llama-cpp/prepare.sh b/backend/cpp/llama-cpp/prepare.sh index bec73a234..1076f48a7 100644 --- a/backend/cpp/llama-cpp/prepare.sh +++ b/backend/cpp/llama-cpp/prepare.sh @@ -109,10 +109,17 @@ if grep -q "llm_add_n_cpu_ffn_overrides" llama.cpp/common/common.h; then else HAS_N_CPU_FFN_HELPER=0 fi +# Newer servers return template state separately from the request JSON. +if grep -q "common_chat_session & out_session" llama.cpp/tools/server/server-common.h; then + HAS_CHAT_SESSION=1 +else + HAS_CHAT_SESSION=0 +fi cat > llama.cpp/tools/grpc-server/llama_compat.h <