From e7cc993b0f2a5179b835e40b6aea3a674675e1eb Mon Sep 17 00:00:00 2001 From: mudler-agent Date: Sat, 17 Oct 2026 22:38:23 +0200 Subject: [PATCH] feat(llama-cpp): add lazy_mode model option (#12743) * feat(llama-cpp): add lazy_mode model option Expose upstream --lazy-mode (on, auto, off) as a model option, for example `lazy_mode:on`. It controls on-demand reading of large tensors such as per-layer embeddings; `on` requires mmap. The option is also accepted as `lazy-mode` or `lzm`. Invalid values log a warning and keep the upstream default (auto). The pinned llama.cpp already provides the lazy_mode field in common_params. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Ettore Di Giacinto * chore(llama-cpp): bump llama.cpp to c0fd78626405 Update the default llama.cpp pin to the current upstream master. The changes since the old pin that the backend touches are additive: a new common_decision_type value, new mtmd audio-stream and download helper declarations. The two server patches still apply in order, and the common_params fields and server headers the backend uses are unchanged. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto Co-authored-by: Claude Sonnet 5.5 --- backend/cpp/llama-cpp/Makefile | 2 +- backend/cpp/llama-cpp/grpc-server.cpp | 18 ++++++++++++++++++ backend/cpp/llama-cpp/prepare.sh | 7 +++++++ docs/content/advanced/model-configuration.md | 20 ++++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index a8d48b618..3bce117b3 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=b76b13379d3b7d67ab0bc424516bb6fd2522bd8e +LLAMA_VERSION?=c0fd78626405c6d9b934efef0ec7291fcc20e3d6 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index f8ba18f40..21e556bef 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -1212,6 +1212,24 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt #endif } } + } else if (!strcmp(optname, "lazy_mode") || !strcmp(optname, "lazy-mode") || !strcmp(optname, "lzm")) { + // On-demand reading of large tensors such as per-layer embeddings + // (upstream -lzm / --lazy-mode). Accepts on, auto or off; "on" needs mmap. +#if LOCALAI_HAS_LAZY_MODE + if (optval != NULL) { + if (optval_str == "on") { + params.lazy_mode = LLAMA_LAZY_MODE_ON; + } else if (optval_str == "auto") { + params.lazy_mode = LLAMA_LAZY_MODE_AUTO; + } else if (optval_str == "off") { + params.lazy_mode = LLAMA_LAZY_MODE_OFF; + } else { + SRV_WRN("lazy_mode: ignoring invalid value '%s' (expected on, auto or off)\n", optval_str.c_str()); + } + } +#else + SRV_WRN("%s", "lazy_mode: this llama.cpp build has no lazy mode, ignoring\n"); +#endif } else if (!strcmp(optname, "n_cpu_moe")) { if (optval != NULL) { try { diff --git a/backend/cpp/llama-cpp/prepare.sh b/backend/cpp/llama-cpp/prepare.sh index cad79e2c3..82d13b4fa 100644 --- a/backend/cpp/llama-cpp/prepare.sh +++ b/backend/cpp/llama-cpp/prepare.sh @@ -122,6 +122,12 @@ if grep -q "moe_cache_size" llama.cpp/common/common.h; then else HAS_MOE_CACHE=0 fi +# On-demand tensor reading (--lazy-mode). Forks that branched earlier lack the field. +if grep -q "llama_lazy_mode lazy_mode" llama.cpp/common/common.h; then + HAS_LAZY_MODE=1 +else + HAS_LAZY_MODE=0 +fi # Newer servers return template state separately from the request JSON. if grep -q "common_chat_session & out_session" llama.cpp/tools/server/server-common.h; then HAS_CHAT_SESSION=1 @@ -137,6 +143,7 @@ cat > llama.cpp/tools/grpc-server/llama_compat.h <