mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 18:09:05 -04:00
0a8a7fbbb4
Bump LLAMA_VERSION to 0d47ea7427463093e69128bf2c2f9cd06b3ee5b3 (73 commits touching common/, src/ and tools/server/). Two upstream changes break the backend: * ggml-org/llama.cpp#20834 folded common_params::use_mmap / use_mlock / use_direct_io into a single `load_mode` enum. LocalAI still exposes the three as independent settings (`mmap`, `mmlock`, and the `direct_io` option), so params_parse folds them once all three have been read, keeping the precedence the separate booleans had: direct I/O bypasses the page cache, mlock implies mmap, everything off is a plain buffered read. turboquant and bonsai compile this same grpc-server.cpp against forks that predate the refactor, so prepare.sh probes the checkout for LLAMA_LOAD_MODE_MMAP and generates llama_compat.h with LOCALAI_LEGACY_LOAD_MODE set accordingly. Probing beats a per-fork build flag here because the fork flavor targets disagree on whether they forward CMAKE_ARGS or EXTRA_CMAKE_ARGS, and it heals itself once a fork rebases past the refactor. * The MiniMax M3 patch no longer applies. Upstream merged the model half of llama.cpp#24523 (LLM_ARCH_MINIMAX_M3, src/models/minimax-m3.cpp, the gguf-py constants and conversion/minimax.py) but not the chat half, so the patch is re-cut to carry only the common/chat.cpp template detection and PEG parser, rebased onto the new pin and onto the thinking_end_tag -> thinking_end_tags rename. Dropping it wholesale (as #11008 did, reverted in #11136) would have silently regressed MiniMax M3 tool calling and thinking. Verified with a CPU docker build of the backend plus LoadModel and Predict against a real GGUF over gRPC in all four load modes. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
226 lines
10 KiB
Diff
226 lines
10 KiB
Diff
# MiniMax-M3 chat-template parser, vendored from upstream llama.cpp PR #24523.
|
|
#
|
|
# Upstream has since merged the *model* half of #24523 (LLM_ARCH_MINIMAX_M3,
|
|
# src/models/minimax-m3.cpp, the gguf-py constants and conversion/minimax.py), so
|
|
# only the chat half is carried here: M3's namespace token "]<]minimax[>[" collides
|
|
# with the autoparser's markup delimiters, so common/chat.cpp needs a dedicated
|
|
# template detection + PEG parser that upstream does not have yet.
|
|
#
|
|
# Rebased against LLAMA_VERSION 0d47ea7427463093e69128bf2c2f9cd06b3ee5b3, which also
|
|
# renamed common_chat_params::thinking_end_tag to thinking_end_tags (a vector).
|
|
# LLAMA_VERSION is auto-bumped nightly; if a bump rejects this patch, re-vendor from
|
|
# #24523 — or, once the chat half merges upstream, delete this file.
|
|
# See https://github.com/mudler/LocalAI/issues/10820 and PR #10837.
|
|
diff --git a/common/chat.cpp b/common/chat.cpp
|
|
index 7a6e7238c..2dd015a2e 100644
|
|
--- a/common/chat.cpp
|
|
+++ b/common/chat.cpp
|
|
@@ -2121,6 +2121,191 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
|
|
return data;
|
|
}
|
|
|
|
+static common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl,
|
|
+ const autoparser::generation_params & inputs) {
|
|
+ common_chat_params data;
|
|
+
|
|
+ data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
|
+ data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
|
+ data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
|
+ data.supports_thinking = true;
|
|
+ data.thinking_start_tag = "<mm:think>";
|
|
+ data.thinking_end_tags = {"</mm:think>"};
|
|
+
|
|
+ // M3 prefixes every tool tag with the namespace token "]<]minimax[>[";
|
|
+ // params use the parameter name as the tag (<file_path>...</file_path>).
|
|
+ const std::string NS = "]<]minimax[>[";
|
|
+ const std::string THINK_START = "<mm:think>";
|
|
+ const std::string THINK_END = "</mm:think>";
|
|
+ const std::string FC_START = NS + "<tool_call>";
|
|
+ const std::string FC_END = NS + "</tool_call>";
|
|
+ const std::string INVOKE_END = NS + "</invoke>";
|
|
+
|
|
+ data.preserved_tokens = {
|
|
+ NS,
|
|
+ "<tool_call>",
|
|
+ "</tool_call>",
|
|
+ THINK_START,
|
|
+ THINK_END,
|
|
+ };
|
|
+
|
|
+ auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
|
+ auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
|
+ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
|
+ auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
|
+
|
|
+ const std::string GEN_PROMPT = data.generation_prompt;
|
|
+
|
|
+ if (inputs.has_continuation()) {
|
|
+ const auto & msg = inputs.continue_msg;
|
|
+
|
|
+ data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
|
+ if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
|
+ data.generation_prompt += THINK_END + msg.render_content();
|
|
+ }
|
|
+
|
|
+ data.prompt += data.generation_prompt;
|
|
+ }
|
|
+
|
|
+ auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
|
+ auto generation_prompt = p.literal(GEN_PROMPT);
|
|
+ auto end = p.end();
|
|
+
|
|
+ auto reasoning = p.eps();
|
|
+ // M3 can emit a bare </mm:think> (no opener) after tool results; keep the opener optional.
|
|
+ if (extract_reasoning && inputs.enable_thinking) {
|
|
+ reasoning = p.optional(p.optional(p.literal(THINK_START)) + p.reasoning(p.until(THINK_END)) + THINK_END);
|
|
+ } else if (extract_reasoning) {
|
|
+ reasoning = p.optional(p.optional(p.literal(THINK_START)) + p.until(THINK_END) + p.literal(THINK_END));
|
|
+ }
|
|
+
|
|
+ if (has_response_format) {
|
|
+ auto response_format = p.rule("response-format",
|
|
+ p.literal("```json") + p.space() +
|
|
+ p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
|
|
+ p.space() + p.literal("```"));
|
|
+ return generation_prompt + reasoning + response_format + end;
|
|
+ }
|
|
+
|
|
+ if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
|
+ return generation_prompt + reasoning + p.content(p.rest()) + end;
|
|
+ }
|
|
+
|
|
+ auto tool_choice = p.choice();
|
|
+ foreach_function(inputs.tools, [&](const json & tool) {
|
|
+ const auto & function = tool.at("function");
|
|
+ std::string name = function.at("name");
|
|
+ auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
|
+ const auto & props = params.contains("properties") ? params.at("properties") : json::object();
|
|
+
|
|
+ std::set<std::string> required;
|
|
+ if (params.contains("required")) {
|
|
+ params.at("required").get_to(required);
|
|
+ }
|
|
+
|
|
+ auto schema_info = common_schema_info();
|
|
+ schema_info.resolve_refs(params);
|
|
+
|
|
+ std::vector<common_peg_parser> required_parsers;
|
|
+ std::vector<common_peg_parser> optional_parsers;
|
|
+ for (const auto & [param_name, param_schema] : props.items()) {
|
|
+ bool is_required = required.find(param_name) != required.end();
|
|
+ bool is_string = schema_info.resolves_to_string(param_schema);
|
|
+
|
|
+ const std::string p_close = NS + "</" + param_name + ">";
|
|
+
|
|
+ auto arg = p.tool_arg(
|
|
+ p.tool_arg_open(
|
|
+ p.literal(NS + "<") +
|
|
+ p.tool_arg_name(p.literal(param_name)) +
|
|
+ p.literal(">")) +
|
|
+ (is_string
|
|
+ ? p.ac(p.tool_arg_string_value(p.until(p_close)) +
|
|
+ p.tool_arg_close(p.literal(p_close)), p_close)
|
|
+ : p.tool_arg_json_value(p.schema(p.json(),
|
|
+ "tool-" + name + "-arg-" + param_name + "-schema",
|
|
+ param_schema, false)) +
|
|
+ p.tool_arg_close(p.literal(p_close))));
|
|
+
|
|
+ auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
|
|
+ if (is_required) {
|
|
+ required_parsers.push_back(named_arg);
|
|
+ } else {
|
|
+ optional_parsers.push_back(named_arg);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ common_peg_parser args_seq = p.eps();
|
|
+ for (size_t i = 0; i < required_parsers.size(); i++) {
|
|
+ if (i > 0) {
|
|
+ args_seq = args_seq + p.space();
|
|
+ }
|
|
+ args_seq = args_seq + required_parsers[i];
|
|
+ }
|
|
+
|
|
+ if (!optional_parsers.empty()) {
|
|
+ common_peg_parser any_opt = p.choice();
|
|
+ for (const auto & opt : optional_parsers) {
|
|
+ any_opt |= opt;
|
|
+ }
|
|
+ args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
|
|
+ }
|
|
+
|
|
+ common_peg_parser invoke_body = args_seq;
|
|
+ auto func_parser = p.tool(
|
|
+ p.tool_open(p.literal(NS + "<invoke name=\"") +
|
|
+ p.tool_name(p.literal(name)) + p.literal("\">")) +
|
|
+ p.space() + invoke_body + p.space() +
|
|
+ p.tool_close(p.literal(INVOKE_END)));
|
|
+
|
|
+ tool_choice |= p.rule("tool-" + name, func_parser);
|
|
+ });
|
|
+
|
|
+ auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
|
+
|
|
+ common_peg_parser tool_calls = p.eps();
|
|
+ if (inputs.parallel_tool_calls) {
|
|
+ tool_calls = p.trigger_rule("tool-call",
|
|
+ p.literal(FC_START) + p.space() + tool_choice +
|
|
+ p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
|
|
+ } else {
|
|
+ tool_calls = p.trigger_rule("tool-call",
|
|
+ p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
|
|
+ }
|
|
+
|
|
+ if (!require_tools) {
|
|
+ tool_calls = p.optional(tool_calls);
|
|
+ }
|
|
+
|
|
+ auto content_before_tools = p.content(p.until(FC_START));
|
|
+ return generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
|
+ });
|
|
+
|
|
+ data.parser = parser.save();
|
|
+
|
|
+ if (include_grammar) {
|
|
+ data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
|
+ data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
|
+ foreach_function(inputs.tools, [&](const json & tool) {
|
|
+ const auto & function = tool.at("function");
|
|
+ auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
|
+ builder.resolve_refs(schema);
|
|
+ });
|
|
+ if (has_response_format) {
|
|
+ auto schema = inputs.json_schema;
|
|
+ builder.resolve_refs(schema);
|
|
+ }
|
|
+ parser.build_grammar(builder, data.grammar_lazy);
|
|
+ });
|
|
+
|
|
+ data.grammar_triggers = {
|
|
+ { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
|
|
+ };
|
|
+ }
|
|
+
|
|
+ return data;
|
|
+}
|
|
+
|
|
// Cohere2 MoE (a.k.a. "North Code") parser.
|
|
//
|
|
// The assistant turn is fully marker-wrapped:
|
|
@@ -2707,6 +2892,15 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
|
|
return common_chat_params_init_gigachat_v3(tmpl, params);
|
|
}
|
|
|
|
+ // MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's
|
|
+ // markup delimiters, so detect the template and use a dedicated parser.
|
|
+ if (src.find("]<]minimax[>[") != std::string::npos &&
|
|
+ src.find("<tool_call>") != std::string::npos &&
|
|
+ src.find("<invoke name=") != std::string::npos) {
|
|
+ LOG_DBG("Using specialized template: MiniMax-M3\n");
|
|
+ return common_chat_params_init_minimax_m3(tmpl, params);
|
|
+ }
|
|
+
|
|
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
|
|
// The template source contains the token as a variable assignment, not as a literal in markup.
|
|
// V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
|