From c68818a62e624a39620f3c6e3473b25401b4e8bf Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Thu, 21 May 2026 11:55:06 +0100 Subject: [PATCH] fix(llama-cpp): terminate tensor_buft_overrides with sentinel (#9919) llama.cpp's model loader asserts back().pattern == nullptr on params.tensor_buft_overrides (and on params.kv_overrides.back().key[0] == 0) before binding them into llama_model_params. PR #8560 attempted to satisfy llama_params_fit's placeholder requirement by pre-filling params.tensor_buft_overrides up to llama_max_tensor_buft_overrides() *before* the option-parse loop. Any subsequent push_back from override_tensor / draft_cpu_moe / draft_n_cpu_moe / draft_override_tensor then appended real entries after the placeholders, leaving back() with a real pattern and tripping the assert. The draft override vector likewise had no terminator at all. Mirror upstream common/arg.cpp:645-658 instead: real entries are pushed during option parsing, and after parsing we pad the main vector up to ntbo (placeholders land at the end, so back() is always nullptr) and append a single {nullptr, nullptr} to the draft vector when it is non-empty. The existing kv_overrides terminator block already matches upstream and stays. Verified against ggml-org/llama.cpp@5cbaa5e: only tensor_buft_overrides (main + draft) and kv_overrides are sentinel-terminated common_params fields; everything else is size-driven std::vector. Assisted-by: claude-code:claude-opus-4-7 Signed-off-by: Richard Palethorpe --- backend/cpp/llama-cpp/grpc-server.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index ce6b47740..61e3f7ee3 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -522,12 +522,6 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt // n_ctx_checkpoints: max context checkpoints per slot (default: 8) params.n_ctx_checkpoints = 8; - // llama memory fit fails if we don't provide a buffer for tensor overrides - const size_t ntbo = llama_max_tensor_buft_overrides(); - while (params.tensor_buft_overrides.size() < ntbo) { - params.tensor_buft_overrides.push_back({nullptr, nullptr}); - } - // decode options. Options are in form optname:optvale, or if booleans only optname. for (int i = 0; i < request->options_size(); i++) { std::string opt = request->options(i); @@ -1081,6 +1075,20 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt params.kv_overrides.back().key[0] = 0; } + // tensor_buft_overrides sentinel termination (mirrors upstream common/arg.cpp). + // Real entries are pushed during option parsing; here we pad/terminate so the + // model loader sees back().pattern == nullptr (GGML_ASSERT at common.cpp:1543) + // and so llama_params_fit has the placeholder slots it requires. + { + const size_t ntbo = llama_max_tensor_buft_overrides(); + while (params.tensor_buft_overrides.size() < ntbo) { + params.tensor_buft_overrides.push_back({nullptr, nullptr}); + } + } + if (!params.speculative.draft.tensor_buft_overrides.empty()) { + params.speculative.draft.tensor_buft_overrides.push_back({nullptr, nullptr}); + } + // TODO: Add yarn if (!request->tensorsplit().empty()) {