From 6e69dbd617abd98a3627ef2e2d7550c9d32a33b3 Mon Sep 17 00:00:00 2001
From: "mudler's LocalAI [bot]"
<139863280+localai-bot@users.noreply.github.com>
Date: Mon, 27 Jul 2026 01:15:06 +0200
Subject: [PATCH] chore: :arrow_up: Update ggml-org/llama.cpp to
`d2a818231effb12b7b20b80b3b8c7756a9a33a04` (#11008)
* :arrow_up: Update ggml-org/llama.cpp
Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
* fix(llama-cpp): drop upstreamed MiniMax M3 patch
The pinned llama.cpp revision already contains MiniMax M3 support, so the downstream patch rejects during backend preparation on every platform.
Assisted-by: Codex:gpt-5
---------
Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>
Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
---
backend/cpp/llama-cpp/Makefile | 2 +-
.../patches/0001-add-minimax-m3-support.patch | 814 ------------------
2 files changed, 1 insertion(+), 815 deletions(-)
delete mode 100644 backend/cpp/llama-cpp/patches/0001-add-minimax-m3-support.patch
diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile
index 8a6bd922c..8964600bf 100644
--- a/backend/cpp/llama-cpp/Makefile
+++ b/backend/cpp/llama-cpp/Makefile
@@ -1,5 +1,5 @@
-LLAMA_VERSION?=571d0d540df04f25298d0e159e520d9fc62ed121
+LLAMA_VERSION?=d2a818231effb12b7b20b80b3b8c7756a9a33a04
LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
CMAKE_ARGS?=
diff --git a/backend/cpp/llama-cpp/patches/0001-add-minimax-m3-support.patch b/backend/cpp/llama-cpp/patches/0001-add-minimax-m3-support.patch
deleted file mode 100644
index c0e66ea50..000000000
--- a/backend/cpp/llama-cpp/patches/0001-add-minimax-m3-support.patch
+++ /dev/null
@@ -1,814 +0,0 @@
-# Vendored from upstream llama.cpp PR #24523 (Preliminary MiniMax-M3 support).
-# Rebased against LLAMA_VERSION 00fa7cb284cbf133fc426733bd64238a3588a33e (also applies cleanly
-# to the later pin 505b1ed15ca80e2a19f12ff4ac365e40fb374053). LLAMA_VERSION is auto-bumped
-# nightly; if a bump rejects this patch, re-vendor from #24523 — or, once #24523 merges
-# upstream, delete this file and bump LLAMA_VERSION normally.
-# See https://github.com/mudler/LocalAI/issues/10820 and PR #10837.
-diff --git a/common/chat.cpp b/common/chat.cpp
-index 22d2ee4..440be9a 100644
---- a/common/chat.cpp
-+++ b/common/chat.cpp
-@@ -2035,6 +2035,191 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
- return data;
- }
-
-+static common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl,
-+ const autoparser::generation_params & inputs) {
-+ common_chat_params data;
-+
-+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
-+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
-+ data.supports_thinking = true;
-+ data.thinking_start_tag = "";
-+ data.thinking_end_tag = "";
-+
-+ // M3 prefixes every tool tag with the namespace token "]<]minimax[>[";
-+ // params use the parameter name as the tag (...).
-+ const std::string NS = "]<]minimax[>[";
-+ const std::string THINK_START = "";
-+ const std::string THINK_END = "";
-+ const std::string FC_START = NS + "";
-+ const std::string FC_END = NS + "";
-+ const std::string INVOKE_END = NS + "";
-+
-+ data.preserved_tokens = {
-+ NS,
-+ "",
-+ "",
-+ THINK_START,
-+ THINK_END,
-+ };
-+
-+ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
-+ auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
-+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-+
-+ const std::string GEN_PROMPT = data.generation_prompt;
-+
-+ if (inputs.has_continuation()) {
-+ const auto & msg = inputs.continue_msg;
-+
-+ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
-+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-+ data.generation_prompt += THINK_END + msg.render_content();
-+ }
-+
-+ data.prompt += data.generation_prompt;
-+ }
-+
-+ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-+ auto generation_prompt = p.literal(GEN_PROMPT);
-+ auto end = p.end();
-+
-+ auto reasoning = p.eps();
-+ // M3 can emit a bare (no opener) after tool results; keep the opener optional.
-+ if (extract_reasoning && inputs.enable_thinking) {
-+ reasoning = p.optional(p.optional(p.literal(THINK_START)) + p.reasoning(p.until(THINK_END)) + THINK_END);
-+ } else if (extract_reasoning) {
-+ reasoning = p.optional(p.optional(p.literal(THINK_START)) + p.until(THINK_END) + p.literal(THINK_END));
-+ }
-+
-+ if (has_response_format) {
-+ auto response_format = p.rule("response-format",
-+ p.literal("```json") + p.space() +
-+ p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
-+ p.space() + p.literal("```"));
-+ return generation_prompt + reasoning + response_format + end;
-+ }
-+
-+ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
-+ return generation_prompt + reasoning + p.content(p.rest()) + end;
-+ }
-+
-+ auto tool_choice = p.choice();
-+ foreach_function(inputs.tools, [&](const json & tool) {
-+ const auto & function = tool.at("function");
-+ std::string name = function.at("name");
-+ auto params = function.contains("parameters") ? function.at("parameters") : json::object();
-+ const auto & props = params.contains("properties") ? params.at("properties") : json::object();
-+
-+ std::set required;
-+ if (params.contains("required")) {
-+ params.at("required").get_to(required);
-+ }
-+
-+ auto schema_info = common_schema_info();
-+ schema_info.resolve_refs(params);
-+
-+ std::vector required_parsers;
-+ std::vector optional_parsers;
-+ for (const auto & [param_name, param_schema] : props.items()) {
-+ bool is_required = required.find(param_name) != required.end();
-+ bool is_string = schema_info.resolves_to_string(param_schema);
-+
-+ const std::string p_close = NS + "";
-+
-+ auto arg = p.tool_arg(
-+ p.tool_arg_open(
-+ p.literal(NS + "<") +
-+ p.tool_arg_name(p.literal(param_name)) +
-+ p.literal(">")) +
-+ (is_string
-+ ? p.ac(p.tool_arg_string_value(p.until(p_close)) +
-+ p.tool_arg_close(p.literal(p_close)), p_close)
-+ : p.tool_arg_json_value(p.schema(p.json(),
-+ "tool-" + name + "-arg-" + param_name + "-schema",
-+ param_schema, false)) +
-+ p.tool_arg_close(p.literal(p_close))));
-+
-+ auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
-+ if (is_required) {
-+ required_parsers.push_back(named_arg);
-+ } else {
-+ optional_parsers.push_back(named_arg);
-+ }
-+ }
-+
-+ common_peg_parser args_seq = p.eps();
-+ for (size_t i = 0; i < required_parsers.size(); i++) {
-+ if (i > 0) {
-+ args_seq = args_seq + p.space();
-+ }
-+ args_seq = args_seq + required_parsers[i];
-+ }
-+
-+ if (!optional_parsers.empty()) {
-+ common_peg_parser any_opt = p.choice();
-+ for (const auto & opt : optional_parsers) {
-+ any_opt |= opt;
-+ }
-+ args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
-+ }
-+
-+ common_peg_parser invoke_body = args_seq;
-+ auto func_parser = p.tool(
-+ p.tool_open(p.literal(NS + "")) +
-+ p.space() + invoke_body + p.space() +
-+ p.tool_close(p.literal(INVOKE_END)));
-+
-+ tool_choice |= p.rule("tool-" + name, func_parser);
-+ });
-+
-+ auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-+
-+ common_peg_parser tool_calls = p.eps();
-+ if (inputs.parallel_tool_calls) {
-+ tool_calls = p.trigger_rule("tool-call",
-+ p.literal(FC_START) + p.space() + tool_choice +
-+ p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
-+ } else {
-+ tool_calls = p.trigger_rule("tool-call",
-+ p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
-+ }
-+
-+ if (!require_tools) {
-+ tool_calls = p.optional(tool_calls);
-+ }
-+
-+ auto content_before_tools = p.content(p.until(FC_START));
-+ return generation_prompt + reasoning + content_before_tools + tool_calls + end;
-+ });
-+
-+ data.parser = parser.save();
-+
-+ if (include_grammar) {
-+ data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
-+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {
-+ foreach_function(inputs.tools, [&](const json & tool) {
-+ const auto & function = tool.at("function");
-+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
-+ builder.resolve_refs(schema);
-+ });
-+ if (has_response_format) {
-+ auto schema = inputs.json_schema;
-+ builder.resolve_refs(schema);
-+ }
-+ parser.build_grammar(builder, data.grammar_lazy);
-+ });
-+
-+ data.grammar_triggers = {
-+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
-+ };
-+ }
-+
-+ return data;
-+}
-+
- // Cohere2 MoE (a.k.a. "North Code") parser.
- //
- // The assistant turn is fully marker-wrapped:
-@@ -2612,6 +2797,15 @@ std::optional common_chat_try_specialized_template(
- return common_chat_params_init_gigachat_v3(tmpl, params);
- }
-
-+ // MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's
-+ // markup delimiters, so detect the template and use a dedicated parser.
-+ if (src.find("]<]minimax[>[") != std::string::npos &&
-+ src.find("") != std::string::npos &&
-+ src.find(" LLM_ARCH_NAMES = {
- { LLM_ARCH_GROVEMOE, "grovemoe" },
- { LLM_ARCH_APERTUS, "apertus" },
- { LLM_ARCH_MINIMAX_M2, "minimax-m2" },
-+ { LLM_ARCH_MINIMAX_M3, "minimax-m3" },
- { LLM_ARCH_COGVLM, "cogvlm" },
- { LLM_ARCH_RND1, "rnd1" },
- { LLM_ARCH_PANGU_EMBED, "pangu-embedded" },
-@@ -395,6 +396,10 @@ static const std::map LLM_TENSOR_NAMES = {
- { LLM_TENSOR_ATTN_POST_NORM, "blk.%d.post_attention_norm" },
- { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
- { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
-+ { LLM_TENSOR_ATTN_INDEX_Q, "blk.%d.attn_index_q" },
-+ { LLM_TENSOR_ATTN_INDEX_K, "blk.%d.attn_index_k" },
-+ { LLM_TENSOR_ATTN_INDEX_Q_NORM, "blk.%d.attn_index_q_norm" },
-+ { LLM_TENSOR_ATTN_INDEX_K_NORM, "blk.%d.attn_index_k_norm" },
- { LLM_TENSOR_ATTN_GATE, "blk.%d.attn_gate" },
- { LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" },
- { LLM_TENSOR_FFN_POST_NORM_1, "blk.%d.post_ffw_norm_1" },
-@@ -761,6 +766,11 @@ static const std::map LLM_TENSOR_INFOS = {
- {LLM_TENSOR_FFN_NORM_EXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
- {LLM_TENSOR_ATTN_Q_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
- {LLM_TENSOR_ATTN_K_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
-+ // minimax-m3 sparse-attn indexer: unused (GGML_OP_NONE) so the loader skips it
-+ {LLM_TENSOR_ATTN_INDEX_Q, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_NONE}},
-+ {LLM_TENSOR_ATTN_INDEX_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_NONE}},
-+ {LLM_TENSOR_ATTN_INDEX_Q_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_NONE}},
-+ {LLM_TENSOR_ATTN_INDEX_K_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_NONE}},
- {LLM_TENSOR_LAYER_OUT_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
- {LLM_TENSOR_LAYER_OUT_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
- {LLM_TENSOR_ATTN_Q_A_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
-@@ -998,6 +1008,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
- case LLM_ARCH_LFM2:
- case LLM_ARCH_LFM2MOE:
- case LLM_ARCH_MINIMAX_M2:
-+ case LLM_ARCH_MINIMAX_M3:
- case LLM_ARCH_MISTRAL4:
- case LLM_ARCH_KIMI_LINEAR:
- return false;
-diff --git a/src/llama-arch.h b/src/llama-arch.h
-index a4f5091..2d50ead 100644
---- a/src/llama-arch.h
-+++ b/src/llama-arch.h
-@@ -144,6 +144,7 @@ enum llm_arch {
- LLM_ARCH_TALKIE,
- LLM_ARCH_MELLUM,
- LLM_ARCH_EAGLE3,
-+ LLM_ARCH_MINIMAX_M3,
- LLM_ARCH_DFLASH,
- LLM_ARCH_UNKNOWN,
- };
-@@ -429,6 +430,10 @@ enum llm_tensor {
- LLM_TENSOR_FFN_LATENT_UP,
- LLM_TENSOR_ATTN_Q_NORM,
- LLM_TENSOR_ATTN_K_NORM,
-+ LLM_TENSOR_ATTN_INDEX_Q, // minimax-m3 sparse-attn indexer (unused)
-+ LLM_TENSOR_ATTN_INDEX_K,
-+ LLM_TENSOR_ATTN_INDEX_Q_NORM,
-+ LLM_TENSOR_ATTN_INDEX_K_NORM,
- LLM_TENSOR_LAYER_OUT_NORM,
- LLM_TENSOR_LAYER_OUT_SCALE,
- LLM_TENSOR_POST_ATTN_NORM,
-diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
-index c8ecb0a..4c2c286 100644
---- a/src/llama-graph.cpp
-+++ b/src/llama-graph.cpp
-@@ -1719,6 +1719,16 @@ ggml_tensor * llm_graph_context::build_ffn(
- cur = ggml_reglu(ctx0, cur);
- cb(cur, "ffn_reglu", il);
- } break;
-+ case LLM_FFN_SWIGLU_OAI:
-+ {
-+ // clamped SwiGLU: parallel gate path (cur=gate, tmp=up)
-+ GGML_ASSERT(gate && type_gate == LLM_FFN_PAR);
-+ constexpr float alpha = 1.702f;
-+ constexpr float limit = 7.0f;
-+ cur = ggml_swiglu_oai(ctx0, cur, tmp, alpha, limit);
-+ cb(cur, "ffn_swiglu_oai", il);
-+ type_gate = LLM_FFN_SEQ; // gate*up already fused; skip the par multiply
-+ } break;
- default:
- GGML_ABORT("fatal error");
- }
-diff --git a/src/llama-graph.h b/src/llama-graph.h
-index c84cb6a..806ce7b 100644
---- a/src/llama-graph.h
-+++ b/src/llama-graph.h
-@@ -54,6 +54,7 @@ enum llm_ffn_op_type : int {
- LLM_FFN_SWIGLU,
- LLM_FFN_GEGLU,
- LLM_FFN_REGLU,
-+ LLM_FFN_SWIGLU_OAI,
- LLM_FFN_SWIGLU_OAI_MOE,
- };
-
-diff --git a/src/llama-model.cpp b/src/llama-model.cpp
-index d874813..7bb71c0 100644
---- a/src/llama-model.cpp
-+++ b/src/llama-model.cpp
-@@ -280,6 +280,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
- return new llama_model_apertus(params);
- case LLM_ARCH_MINIMAX_M2:
- return new llama_model_minimax_m2(params);
-+ case LLM_ARCH_MINIMAX_M3:
-+ return new llama_model_minimax_m3(params);
- case LLM_ARCH_COGVLM:
- return new llama_model_cogvlm(params);
- case LLM_ARCH_PANGU_EMBED:
-@@ -807,6 +809,7 @@ const char * llm_type_name(llm_type type) {
- case LLM_TYPE_310B_A15B: return "310B.A15B";
- case LLM_TYPE_355B_A32B: return "355B.A32B";
- case LLM_TYPE_397B_A17B: return "397B.A17B";
-+ case LLM_TYPE_428B_A23B: return "428B.A23B";
- case LLM_TYPE_685B_A37B: return "685B.A37B";
- case LLM_TYPE_744B_A40B: return "744B.A40B";
- case LLM_TYPE_E2B: return "E2B";
-@@ -2532,6 +2535,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
- case LLM_ARCH_GROVEMOE:
- case LLM_ARCH_APERTUS:
- case LLM_ARCH_MINIMAX_M2:
-+ case LLM_ARCH_MINIMAX_M3:
- case LLM_ARCH_COGVLM:
- case LLM_ARCH_PANGU_EMBED:
- case LLM_ARCH_AFMOE:
-diff --git a/src/llama-model.h b/src/llama-model.h
-index 45b054c..540e0d2 100644
---- a/src/llama-model.h
-+++ b/src/llama-model.h
-@@ -139,6 +139,7 @@ enum llm_type {
- LLM_TYPE_310B_A15B, // /MiMo-V2-Flash
- LLM_TYPE_355B_A32B, // GLM-4.5
- LLM_TYPE_397B_A17B, // Qwen3.5
-+ LLM_TYPE_428B_A23B, // MiniMax M3
- LLM_TYPE_685B_A37B, // DeepSeek V3.2
- LLM_TYPE_744B_A40B, // GLM-5
- LLM_TYPE_E2B,
-diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp
-new file mode 100644
-index 0000000..137852a
---- /dev/null
-+++ b/src/models/minimax-m3.cpp
-@@ -0,0 +1,197 @@
-+#include "models.h"
-+
-+// MiniMax-M3, text-only: MiniMax-M2 GQA (per-head QK-norm, partial rotary) + DeepSeek-V3
-+// leading-dense/routed/shared experts (swigluoai). Sparse attn -> dense; vision + MTP dropped.
-+
-+void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {
-+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-+ ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);
-+ ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
-+ ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
-+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
-+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
-+ ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
-+
-+ switch (hparams.n_layer()) {
-+ case 60: type = LLM_TYPE_428B_A23B; break;
-+ default: type = LLM_TYPE_UNKNOWN;
-+ }
-+}
-+
-+void llama_model_minimax_m3::load_arch_tensors(llama_model_loader &) {
-+ LLAMA_LOAD_LOCALS;
-+ const int64_t n_expert_shared = hparams.n_expert_shared;
-+ const int64_t n_ff_exp = hparams.n_ff_exp;
-+
-+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
-+
-+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
-+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);
-+
-+ for (int i = 0; i < n_layer; ++i) {
-+ auto & layer = layers[i];
-+
-+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
-+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0);
-+
-+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
-+ // per-head QK-norm (one head_dim vector)
-+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
-+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
-+
-+ // sparse-attn indexer (unused): GGML_OP_NONE -> loader skips; NOT_REQUIRED -> older GGUFs still load;
-+ // SKIP_IF_VIRTUAL -> no-file loader (test-llama-archs) skips them too
-+ const int64_t n_index_head = 4; // sparse_num_index_heads
-+ const int64_t d_index = 128; // sparse_index_dim
-+ const int idx_flags = TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL;
-+ create_tensor(tn(LLM_TENSOR_ATTN_INDEX_Q, "weight", i), {n_embd, n_index_head * d_index}, idx_flags);
-+ create_tensor(tn(LLM_TENSOR_ATTN_INDEX_K, "weight", i), {n_embd, d_index}, idx_flags);
-+ create_tensor(tn(LLM_TENSOR_ATTN_INDEX_Q_NORM, "weight", i), {d_index}, idx_flags);
-+ create_tensor(tn(LLM_TENSOR_ATTN_INDEX_K_NORM, "weight", i), {d_index}, idx_flags);
-+
-+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
-+
-+ if (i < (int) hparams.n_layer_dense_lead) {
-+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
-+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
-+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
-+ } else {
-+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
-+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
-+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
-+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
-+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
-+
-+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
-+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, 0);
-+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
-+ }
-+ }
-+}
-+
-+std::unique_ptr llama_model_minimax_m3::build_arch_graph(const llm_graph_params & params) const {
-+ return std::make_unique(*this, params);
-+}
-+
-+llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
-+ const int64_t n_embd_head = hparams.n_embd_head_v();
-+
-+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
-+ // partial rotary: head_dim != n_rot, so don't assert n_embd_head == n_rot
-+
-+ ggml_tensor * cur;
-+ ggml_tensor * inpL;
-+
-+ inpL = build_inp_embd(model.tok_embd);
-+
-+ ggml_tensor * inp_pos = build_inp_pos();
-+ auto inp_attn = build_attn_inp_kv();
-+ ggml_tensor * inp_out_ids = build_inp_out_ids();
-+
-+ for (int il = 0; il < n_layer; ++il) {
-+ ggml_tensor * inpSA = inpL;
-+
-+ // self-attention
-+ {
-+ cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
-+ cb(cur, "attn_norm", il);
-+
-+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
-+ n_embd_head, n_head, n_head_kv, il);
-+
-+ // per-head QK RMSNorm (weights include Gemma +1)
-+ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
-+ cb(Qcur, "Qcur_normed", il);
-+ Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
-+ cb(Kcur, "Kcur_normed", il);
-+
-+ Qcur = ggml_rope_ext(
-+ ctx0, Qcur, inp_pos, nullptr,
-+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-+ ext_factor, attn_factor, beta_fast, beta_slow
-+ );
-+ Kcur = ggml_rope_ext(
-+ ctx0, Kcur, inp_pos, nullptr,
-+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-+ ext_factor, attn_factor, beta_fast, beta_slow
-+ );
-+
-+ cb(Qcur, "Qcur", il);
-+ cb(Kcur, "Kcur", il);
-+ cb(Vcur, "Vcur", il);
-+
-+ cur = build_attn(inp_attn,
-+ model.layers[il].wo, NULL, model.layers[il].wo_s,
-+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
-+ }
-+
-+ if (il == n_layer - 1 && inp_out_ids) {
-+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
-+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
-+ }
-+
-+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
-+ cb(ffn_inp, "ffn_inp", il);
-+
-+ cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
-+ cb(cur, "ffn_norm", il);
-+
-+ if ((uint32_t) il < hparams.n_layer_dense_lead) {
-+ // leading dense
-+ cur = build_ffn(cur,
-+ model.layers[il].ffn_up, NULL, NULL,
-+ model.layers[il].ffn_gate, NULL, NULL,
-+ model.layers[il].ffn_down, NULL, NULL,
-+ NULL,
-+ LLM_FFN_SWIGLU_OAI, LLM_FFN_PAR, il);
-+ cb(cur, "ffn_out", il);
-+ } else {
-+ // routed experts
-+ ggml_tensor * moe_out = build_moe_ffn(cur,
-+ model.layers[il].ffn_gate_inp,
-+ model.layers[il].ffn_up_exps,
-+ model.layers[il].ffn_gate_exps,
-+ model.layers[il].ffn_down_exps,
-+ model.layers[il].ffn_exp_probs_b,
-+ n_expert, n_expert_used,
-+ LLM_FFN_SWIGLU_OAI_MOE, hparams.expert_weights_norm,
-+ hparams.expert_weights_scale,
-+ (llama_expert_gating_func_type) hparams.expert_gating_func,
-+ il);
-+ cb(moe_out, "ffn_moe_out", il);
-+
-+ // shared expert
-+ ggml_tensor * ffn_shexp = build_ffn(cur,
-+ model.layers[il].ffn_up_shexp, NULL, NULL,
-+ model.layers[il].ffn_gate_shexp, NULL, NULL,
-+ model.layers[il].ffn_down_shexp, NULL, NULL,
-+ NULL,
-+ LLM_FFN_SWIGLU_OAI, LLM_FFN_PAR, il);
-+ cb(ffn_shexp, "ffn_shexp", il);
-+
-+ cur = ggml_add(ctx0, moe_out, ffn_shexp);
-+ cb(cur, "ffn_out", il);
-+ }
-+
-+ cur = ggml_add(ctx0, cur, ffn_inp);
-+
-+ cur = build_cvec(cur, il);
-+ cb(cur, "l_out", il);
-+
-+ // input for next layer
-+ inpL = cur;
-+ }
-+
-+ cur = inpL;
-+
-+ cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
-+ cb(cur, "result_norm", -1);
-+ res->t_embd = cur;
-+
-+ // lm_head
-+ cur = build_lora_mm(model.output, cur, model.output_s);
-+ cb(cur, "result_output", -1);
-+ res->t_logits = cur;
-+
-+ ggml_build_forward_expand(gf, cur);
-+}
-diff --git a/src/models/models.h b/src/models/models.h
-index 7a52e7b..5e2a826 100644
---- a/src/models/models.h
-+++ b/src/models/models.h
-@@ -1870,6 +1870,17 @@ struct llama_model_minimax_m2 : public llama_model_base {
- std::unique_ptr build_arch_graph(const llm_graph_params & params) const override;
- };
-
-+struct llama_model_minimax_m3 : public llama_model_base {
-+ llama_model_minimax_m3(const struct llama_model_params & params) : llama_model_base(params) {}
-+ void load_arch_hparams(llama_model_loader & ml) override;
-+ void load_arch_tensors(llama_model_loader & ml) override;
-+
-+ struct graph : public llm_graph_context {
-+ graph(const llama_model & model, const llm_graph_params & params);
-+ };
-+
-+ std::unique_ptr build_arch_graph(const llm_graph_params & params) const override;
-+};
-
- struct llama_model_cogvlm : public llama_model_base {
- llama_model_cogvlm(const struct llama_model_params & params) : llama_model_base(params) {}
-diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
-index f39abe7..2085f43 100644
---- a/tests/test-llama-archs.cpp
-+++ b/tests/test-llama-archs.cpp
-@@ -352,6 +352,7 @@ static bool moe_mandatory(const llm_arch arch) {
- case LLM_ARCH_LLADA_MOE:
- case LLM_ARCH_GROVEMOE:
- case LLM_ARCH_MINIMAX_M2:
-+ case LLM_ARCH_MINIMAX_M3:
- case LLM_ARCH_RND1:
- case LLM_ARCH_PADDLEOCR:
- case LLM_ARCH_MIMO2: