chore: ⬆️ Update ggml-org/llama.cpp to e70802a01f03f0ed31a26338a5664796f3824371 (#11755)

* ⬆️ Update ggml-org/llama.cpp

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* fix(llama-cpp): follow upstream MTMD APIs

The dependency update adds MTMD initialization options to prompt and
bitmap helpers. The gRPC adapter now passes the server options through
each affected path.

The update also replaces the per-layer MoE regex helper. Preparation
probes both APIs because older forks still reuse this adapter.

Assisted-by: Codex:gpt-5

---------

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>
Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=925e1179947ea0c0ebfb0032df18af3a729822be
+LLAMA_VERSION?=e70802a01f03f0ed31a26338a5664796f3824371
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -88,6 +88,12 @@ using grpc::ServerBuilder;
 using grpc::ServerContext;
 using grpc::Status;
 
+#if LOCALAI_HAS_MTMD_INIT_OPT
+#define LOCALAI_MTMD_INIT_OPT_ARG(value) , value
+#else
+#define LOCALAI_MTMD_INIT_OPT_ARG(value)
+#endif
+
 // gRPC bearer token auth for distributed mode.
 // Reads LOCALAI_GRPC_AUTH_TOKEN from the environment. When set, rejects
 // requests without a matching "authorization: Bearer <token>" metadata header.
@@ -1119,14 +1125,16 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
                 try {
                     int n = std::stoi(optval_str);
                     if (n < 0) n = 0;
-                    // Keep override-name storage alive for the lifetime of the params struct
-                    // (mirrors upstream arg.cpp behavior with a function-local static).
+#if LOCALAI_HAS_N_CPU_FFN_HELPER
+                    llm_add_n_cpu_ffn_overrides(n, LLM_FFN_EXPS_REGEX, params.speculative.draft.tensor_buft_overrides);
+#else
                     static std::list<std::string> buft_overrides_draft;
                     for (int i = 0; i < n; ++i) {
                         buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i));
                         params.speculative.draft.tensor_buft_overrides.push_back(
                             {buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()});
                     }
+#endif
                 } catch (...) {}
             }
 
@@ -1144,14 +1152,16 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
                 try {
                     int n = std::stoi(optval_str);
                     if (n < 0) n = 0;
-                    // Keep override-name storage alive for the lifetime of the
-                    // params struct (mirrors upstream arg.cpp's function-local static).
+#if LOCALAI_HAS_N_CPU_FFN_HELPER
+                    llm_add_n_cpu_ffn_overrides(n, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides);
+#else
                     static std::list<std::string> buft_overrides_main;
                     for (int i = 0; i < n; ++i) {
                         buft_overrides_main.push_back(llm_ffn_exps_block_regex(i));
                         params.tensor_buft_overrides.push_back(
                             {buft_overrides_main.back().c_str(), ggml_backend_cpu_buffer_type()});
                     }
+#endif
                 } catch (...) {}
             }
 
@@ -2111,10 +2121,10 @@ public:
             std::vector<server_tokens> inputs;
             if (has_mtmd) {
                 // multimodal
-                inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files));
+                inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)));
             } else {
                  // Everything else, including multimodal completions.
-                inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true);
+                inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
             }
 
             tasks.reserve(inputs.size());
@@ -2686,10 +2696,10 @@ public:
             std::vector<server_tokens> inputs;
             if (has_mtmd) {
                 // multimodal
-                inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files));
+                inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)));
             } else {
                  // Everything else, including multimodal completions.
-                inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true);
+                inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
             }
 
             tasks.reserve(inputs.size());
@@ -2876,7 +2886,7 @@ public:
         json prompt = body.at("embeddings");
 
 
-        auto tokenized_prompts = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt, true, true);
+        auto tokenized_prompts = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
         for (const auto & tokens : tokenized_prompts) {
             // this check is necessary for models that do not add BOS token to the input
             if (tokens.empty()) {
@@ -2981,7 +2991,7 @@ public:
 
             tasks.reserve(documents.size());
             for (size_t i = 0; i < documents.size(); i++) {
-                auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i]);
+                auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i] LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
                 server_task task = server_task(SERVER_TASK_TYPE_RERANK);
                 task.id = rd.queue_tasks.get_new_id();
                 task.index = i;
@@ -3062,7 +3072,7 @@ public:
             return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, opts.error);
         }
 
-        auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false);
+        auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
         if (!wrapper.bitmap) {
             return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
                 "failed to read speaker reference audio: " + opts.voice_path);

backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch

@@ -659,7 +659,7 @@ index 9069463fe..b7fa1e534 100644
 +        }
 +
 +        if (speaker_ref_len > 0) {
-+            auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false);
++            auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
 +            if (!wrapper.bitmap) {
 +                res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
 +                return res;

backend/cpp/llama-cpp/prepare.sh

@@ -61,11 +61,23 @@ if grep -q "server_metrics metrics;" llama.cpp/tools/server/server-task.h; then
 else
     HAS_SERVER_METRICS=0
 fi
+if grep -q "mtmd_helper_init_opt" llama.cpp/tools/mtmd/mtmd-helper.h; then
+    HAS_MTMD_INIT_OPT=1
+else
+    HAS_MTMD_INIT_OPT=0
+fi
+if grep -q "llm_add_n_cpu_ffn_overrides" llama.cpp/common/common.h; then
+    HAS_N_CPU_FFN_HELPER=1
+else
+    HAS_N_CPU_FFN_HELPER=0
+fi
 cat > llama.cpp/tools/grpc-server/llama_compat.h <<EOF
 // Generated by backend/cpp/llama-cpp/prepare.sh. Do not edit.
 #pragma once
 #define LOCALAI_LEGACY_LOAD_MODE ${LEGACY_LOAD_MODE}
 #define LOCALAI_HAS_SERVER_METRICS ${HAS_SERVER_METRICS}
+#define LOCALAI_HAS_MTMD_INIT_OPT ${HAS_MTMD_INIT_OPT}
+#define LOCALAI_HAS_N_CPU_FFN_HELPER ${HAS_N_CPU_FFN_HELPER}
 EOF
 
 set +e