diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 5c02e597b..0fdac506b 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=f2cc7282ce0042641df55e0e563b75e824e2b647 +LLAMA_VERSION?=a0e6dc7384edb8f5d927155e1d26c10b4a310e6d LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index f6bc0f34d..cc27a40f6 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -108,6 +108,12 @@ using grpc::Status; // requests without a matching "authorization: Bearer " metadata header. // Cached auth token — empty means auth is disabled. +// Newer llama.cpp returns common_chat_params::prompt as a common_chat_prompt +// (use .str()); older forks such as turboquant still return std::string. +static inline std::string chat_prompt_str(const std::string & p) { return p; } +template +static inline std::string chat_prompt_str(const T & p) { return p.str(); } + static std::string g_grpc_auth_token; // Minimal constant-time comparison (avoids OpenSSL dependency) @@ -3823,7 +3829,7 @@ public: dummy_inputs.use_jinja = params_base.use_jinja; const auto rendered = common_chat_templates_apply(ctx_server.impl->chat_params.tmpls.get(), dummy_inputs); - rendered_template = rendered.prompt; + rendered_template = chat_prompt_str(rendered.prompt); } response->set_rendered_template(rendered_template);