mirror/LocalAI
1
0 Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-09-14 07:07:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore: ⬆️ Update ggml-org/llama.cpp to f280b26983ad0fdb705a0d9ebf0503e76f2899b0 (#11646)

Browse Source 
* ⬆️ Update ggml-org/llama.cpp

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* fix(llama-cpp): adapt to the common JSON API

The llama.cpp bump replaces its nlohmann JSON alias with common_json. Update the gRPC adapter for the new exception, iterator, conversion, and container APIs.

Assisted-by: Codex:gpt-5.6 [systematic-debugging]

* fix(turboquant): adapt the JSON exception type

The shared gRPC source now follows the upstream common_json API. The
TurboQuant fork still exposes nlohmann JSON and cannot compile the new
exception type.

Translate that exception in the fork-specific source patch so both
llama.cpp variants compile from the shared adapter.

Assisted-by: Codex:gpt-5.6 [systematic-debugging]

* fix(bonsai): adapt the JSON exception type

The shared gRPC source uses upstream's common_json wrapper. The Bonsai fork still exposes nlohmann JSON and cannot compile that exception type.\n\nTranslate the exception in the fork-specific preparation step and verify that repeated preparation stays idempotent.\n\nAssisted-by: Codex:gpt-5.6 [systematic-debugging]

* fix(llama-cpp): let prepare register gRPC

The score patch duplicated the gRPC CMake registration that prepare.sh already owns. Its stale context rejects the current upstream tools file on Darwin before compilation starts.

Assisted-by: Codex:gpt-5

---------

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>
Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
This commit is contained in:
mudler's LocalAI [bot]mudlerlocalai-org-maint-bot
 authored and GitHub committed 2026-08-25 12:57:12 +02:00
1 parent ccb9a0a088
commit fa9ffc181c
8 files changed
+97 -44
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/cpp/llama-cpp/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
LLAMA_VERSION?=d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd
LLAMA_VERSION?=f280b26983ad0fdb705a0d9ebf0503e76f2899b0
LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
CMAKE_ARGS?=
backend/cpp/llama-cpp/grpc-server.cpp
+26 -29
View File
@@ -294,7 +294,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
} else {
SRV_WRN("[TOOLS DEBUG] parse_options: Parsed tools JSON is not an array: %s\n", tools_json.dump().c_str());
}
} catch (const json::parse_error& e) {
} catch (const common_json_error& e) {
SRV_WRN("Failed to parse tools JSON from proto: %s\n", e.what());
SRV_WRN("[TOOLS DEBUG] parse_options: Tools string that failed to parse: %s\n", predict->tools().c_str());
}
@@ -324,7 +324,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
SRV_DBG("[TOOLS DEBUG] Received tool_choice object from Go layer: %s\n", tool_choice_json.dump().c_str());
}
SRV_INF("Extracted tool_choice from proto: %s\n", predict->toolchoice().c_str());
} catch (const json::parse_error& e) {
} catch (const common_json_error& e) {
// If parsing fails, treat as string
data["tool_choice"] = predict->toolchoice();
SRV_INF("Extracted tool_choice as string: %s\n", predict->toolchoice().c_str());
@@ -353,7 +353,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
// Add to data - llama.cpp server expects it as an object (map)
data["logit_bias"] = logit_bias_json;
SRV_INF("Using logit_bias: %s\n", predict->logitbias().c_str());
} catch (const json::parse_error& e) {
} catch (const common_json_error& e) {
SRV_ERR("Failed to parse logit_bias JSON from proto: %s\n", e.what());
}
}
@@ -398,7 +398,10 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
});
}
data["stop"] = predict->stopprompts();
data["stop"] = json::array();
for (const auto & stop : predict->stopprompts()) {
data["stop"].push_back(stop);
}
// data["n_probs"] = predict->nprobs();
//TODO: images,
@@ -1795,7 +1798,7 @@ public:
for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j));
for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j));
}
messages_json.push_back(llama_grpc::build_reconstructed_message(rin));
messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump()));
}
// Final safety check: Ensure no message has null content (Jinja templates require strings)
@@ -1988,7 +1991,7 @@ public:
if (!body_json.contains("chat_template_kwargs")) {
body_json["chat_template_kwargs"] = json::object();
}
for (auto& el : ctk.items()) {
for (auto el : ctk.items()) {
body_json["chat_template_kwargs"][el.key()] = el.value();
}
}
@@ -2074,30 +2077,27 @@ public:
// If not using chat templates, extract files from image_data/audio_data fields
// (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
const auto &images_data = data.find("image_data");
if (images_data != data.end() && images_data->is_array())
if (data.contains("image_data") && data.at("image_data").is_array())
{
for (const auto &img : *images_data)
for (const auto &img : data.at("image_data"))
{
auto decoded_data = base64_decode(img["data"].get<std::string>());
files.push_back(decoded_data);
}
}
const auto &audio_data = data.find("audio_data");
if (audio_data != data.end() && audio_data->is_array())
if (data.contains("audio_data") && data.at("audio_data").is_array())
{
for (const auto &audio : *audio_data)
for (const auto &audio : data.at("audio_data"))
{
auto decoded_data = base64_decode(audio["data"].get<std::string>());
files.push_back(decoded_data);
}
}
const auto &video_data = data.find("video_data");
if (video_data != data.end() && video_data->is_array())
if (data.contains("video_data") && data.at("video_data").is_array())
{
for (const auto &video : *video_data)
for (const auto &video : data.at("video_data"))
{
auto decoded_data = base64_decode(video["data"].get<std::string>());
files.push_back(decoded_data);
@@ -2370,7 +2370,7 @@ public:
for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j));
for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j));
}
messages_json.push_back(llama_grpc::build_reconstructed_message(rin));
messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump()));
}
// Final safety check: Ensure no message has null content (Jinja templates require strings)
@@ -2563,7 +2563,7 @@ public:
if (!body_json.contains("chat_template_kwargs")) {
body_json["chat_template_kwargs"] = json::object();
}
for (auto& el : ctk.items()) {
for (auto el : ctk.items()) {
body_json["chat_template_kwargs"][el.key()] = el.value();
}
}
@@ -2649,11 +2649,10 @@ public:
// If not using chat templates, extract files from image_data/audio_data fields
// (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
const auto &images_data = data.find("image_data");
if (images_data != data.end() && images_data->is_array())
if (data.contains("image_data") && data.at("image_data").is_array())
{
std::cout << "[PREDICT] Processing " << images_data->size() << " images" << std::endl;
for (const auto &img : *images_data)
std::cout << "[PREDICT] Processing " << data.at("image_data").size() << " images" << std::endl;
for (const auto &img : data.at("image_data"))
{
std::cout << "[PREDICT] Processing image" << std::endl;
auto decoded_data = base64_decode(img["data"].get<std::string>());
@@ -2661,20 +2660,18 @@ public:
}
}
const auto &audio_data = data.find("audio_data");
if (audio_data != data.end() && audio_data->is_array())
if (data.contains("audio_data") && data.at("audio_data").is_array())
{
for (const auto &audio : *audio_data)
for (const auto &audio : data.at("audio_data"))
{
auto decoded_data = base64_decode(audio["data"].get<std::string>());
files.push_back(decoded_data);
}
}
const auto &video_data = data.find("video_data");
if (video_data != data.end() && video_data->is_array())
if (data.contains("video_data") && data.at("video_data").is_array())
{
for (const auto &video : *video_data)
for (const auto &video : data.at("video_data"))
{
auto decoded_data = base64_decode(video["data"].get<std::string>());
files.push_back(decoded_data);
@@ -3005,7 +3002,7 @@ public:
}
// Collect responses
json responses = json::array();
std::vector<json> responses;
for (auto & res : all_results.results) {
GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);
responses.push_back(res->to_json());
@@ -3018,7 +3015,7 @@ public:
// Crop results by request.top_n if specified
int top_n = request->top_n();
if (top_n > 0 && top_n < static_cast<int>(responses.size())) {
responses = json(responses.begin(), responses.begin() + top_n);
responses.resize(top_n);
}
// Set usage information
backend::Usage* usage = rerankResult->mutable_usage();
backend/cpp/llama-cpp/message_content.h
+4 -3
View File
@@ -52,14 +52,15 @@ inline nlohmann::ordered_json normalize_message_content(const std::string& role,
// (#7528). A multimodal user message legitimately carries a typed-part array
// ({type:text}, {type:image_url}, ...), which must be left intact. Shared by the
// streaming and non-streaming paths so this invariant cannot drift between them.
inline void normalize_template_message(nlohmann::ordered_json& msg) {
template <typename Json>
inline void normalize_template_message(Json& msg) {
if (!msg.contains("content")) {
msg["content"] = ""; // templates expect the field to exist
return;
}
nlohmann::ordered_json& content = msg["content"];
auto& content = msg["content"];
const std::string role = (msg.contains("role") && msg["role"].is_string())
? msg["role"].get<std::string>()
? msg["role"].template get<std::string>()
: std::string();
if (content.is_null()) {
content = ""; // #7324: null would crash content[:N] slicing
backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch
+1 -11
View File
@@ -6,10 +6,9 @@ Subject: [PATCH 1/2] score-patch
---
common/common.cpp | 6 +-
common/common.h | 3 +
tools/CMakeLists.txt | 1 +
tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++-
tools/server/server-task.h | 47 +++++
5 files changed, 406 insertions(+), 9 deletions(-)
4 files changed, 405 insertions(+), 9 deletions(-)
diff --git a/common/common.cpp b/common/common.cpp
index 2e3f14c..0cec0dc 100644
@@ -42,15 +41,6 @@ index 878534d..4001df2 100644
int32_t n_sequences = 1; // number of sequences to decode
int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch)
int32_t n_outputs_max_per_seq = 1; // max outputs per sequence
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index 780df32..1d2fe8f 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -41,3 +41,4 @@ else()
add_subdirectory(fit-params)
add_subdirectory(results)
endif()
+add_subdirectory(grpc-server)
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 3b5f6a1..d0e18e6 100644
--- a/tools/server/server-context.cpp