package config import ( "context" "github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/reasoning" "github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/xlog" gguf "github.com/gpustack/gguf-parser-go" "github.com/gpustack/gguf-parser-go/util/ptr" ) // reservedNonChatModel reports whether the operator reserved this model for an // internal primitive — the router score classifier or the PII NER // token_classify tier. Such a model has no chat template and must not be // given the generative-chat defaults the GGUF importer otherwise applies // (FLAG_CHAT, jinja templating): surfacing it in chat pickers defeats the // reservation. Operators who do want a combined model declare both usecases // explicitly — the combination is valid. func reservedNonChatModel(cfg *ModelConfig) bool { return cfg.KnownUsecases != nil && (*cfg.KnownUsecases&(FLAG_SCORE|FLAG_TOKEN_CLASSIFY)) != 0 } func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) { if defaultCtx == 0 && cfg.ContextSize == nil { // trainedMax is the model's full trained context window (n_ctx_train). // Defaulting a model to it unbounded is what OOMs long-context models at // load: a 128k / 256k / 1M KV cache cannot fit a consumer GPU and the // backend aborts (exitCode=-1). autoContextSize instead caps to a modest // default and only steps below it when detected per-device VRAM demands. trainedMax := int(f.EstimateLLaMACppRun().ContextSize) if trainedMax > 0 { cSize := autoContextSize(f, trainedMax) cfg.ContextSize = &cSize } else { defaultCtx = DefaultContextSize cfg.ContextSize = &defaultCtx } } // GPU options if cfg.Options == nil { if xsysinfo.HasGPU("nvidia") || xsysinfo.HasGPU("amd") { cfg.Options = []string{"gpu"} } } if cfg.NGPULayers == nil { // we assume we want to offload all layers defaultHigh := DefaultNGPULayers cfg.NGPULayers = &defaultHigh } xlog.Debug("[gguf] guessDefaultsFromFile: NGPULayers set", "NGPULayers", cfg.NGPULayers, "modelName", f.Metadata().Name) // identify from well known templates first, otherwise use the raw jinja template chatTemplate, found := f.Header.MetadataKV.Get("tokenizer.chat_template") if found { // fill jinja template cfg.modelTemplate = chatTemplate.ValueString() } // Auto-enable Multi-Token Prediction (ggml-org/llama.cpp#22673) when the // GGUF carries an embedded MTP head. Skipped silently for non-MTP models // and when the user already configured a spec_type. if n, ok := HasEmbeddedMTPHead(f); ok { ApplyMTPDefaults(cfg, n) } // Thinking support detection is done after model load via DetectThinkingSupportFromBackend // template estimations if cfg.HasTemplate() { // nothing to guess here xlog.Debug("[gguf] guessDefaultsFromFile: template already set", "name", cfg.Name, "modelName", f.Metadata().Name) return } xlog.Debug("[gguf] Model file loaded", "file", cfg.ModelFileName(), "eosTokenID", f.Tokenizer().EOSTokenID, "bosTokenID", f.Tokenizer().BOSTokenID, "modelName", f.Metadata().Name, "architecture", f.Architecture().Architecture) // guess the name if cfg.Name == "" { cfg.Name = f.Metadata().Name } // A model the operator reserved for an internal primitive (the router // score classifier, or the PII NER token_classify tier) is not a chat // model: it carries no chat template and must not be painted with the // generative-chat defaults — appending FLAG_CHAT here would fold chat // into KnownUsecases on the next sync and surface the model in every // chat picker. Respect the declaration. if !reservedNonChatModel(cfg) { // Instruct to use template from llama.cpp cfg.TemplateConfig.UseTokenizerTemplate = true cfg.FunctionsConfig.GrammarConfig.NoGrammar = true cfg.Options = append(cfg.Options, "use_jinja:true") cfg.KnownUsecaseStrings = append(cfg.KnownUsecaseStrings, "FLAG_CHAT") } // Apply per-model-family inference parameter defaults (temperature, top_p, etc.) ApplyInferenceDefaults(cfg, f.Metadata().Name) } // DetectThinkingSupportFromBackend calls the ModelMetadata gRPC method to detect // if the model supports thinking mode and if the template ends with a thinking start token. // This should be called after the model is loaded. // The results are stored in cfg.SupportsThinking and cfg.ThinkingForcedOpen. // The backend-reported multimodal marker is also captured into cfg.MediaMarker. func DetectThinkingSupportFromBackend(ctx context.Context, cfg *ModelConfig, backendClient grpc.Backend, modelOptions *pb.ModelOptions) { if backendClient == nil { xlog.Debug("[gguf] DetectThinkingSupportFromBackend: backend client is nil, skipping detection") return } if modelOptions == nil { xlog.Debug("[gguf] DetectThinkingSupportFromBackend: model options is nil, skipping detection") return } // Only llama-cpp exposes ModelMetadata today. Other backends will either error // or return an empty response — both are fine, we just bail before calling. if cfg.Backend != "llama-cpp" { xlog.Debug("[gguf] DetectThinkingSupportFromBackend: skipping detection", "backend", cfg.Backend) return } metadata, err := backendClient.ModelMetadata(ctx, modelOptions) if err != nil { xlog.Warn("[gguf] DetectThinkingSupportFromBackend: failed to get model metadata", "error", err) return } if metadata != nil { // The multimodal media marker is backend-controlled (llama.cpp may pick a // random per-server string). Empty means "no mtmd context" — Go falls back // to templates.DefaultMultiMediaMarker at render time. if metadata.MediaMarker != "" { cfg.MediaMarker = metadata.MediaMarker xlog.Debug("[gguf] DetectThinkingSupportFromBackend: media marker captured", "marker", metadata.MediaMarker) } // Thinking / tool-format detection only applies when we rely on the // backend-side tokenizer template — otherwise the rendered-template based // heuristics below aren't meaningful. if !cfg.TemplateConfig.UseTokenizerTemplate { return } applyDetectedThinkingConfig(cfg, metadata) // Extract tool format markers from autoparser analysis if tf := metadata.GetToolFormat(); tf != nil && tf.FormatType != "" { cfg.FunctionsConfig.ToolFormatMarkers = &functions.ToolFormatMarkers{ FormatType: tf.FormatType, SectionStart: tf.SectionStart, SectionEnd: tf.SectionEnd, PerCallStart: tf.PerCallStart, PerCallEnd: tf.PerCallEnd, FuncNamePrefix: tf.FuncNamePrefix, FuncNameSuffix: tf.FuncNameSuffix, FuncClose: tf.FuncClose, ArgNamePrefix: tf.ArgNamePrefix, ArgNameSuffix: tf.ArgNameSuffix, ArgValuePrefix: tf.ArgValuePrefix, ArgValueSuffix: tf.ArgValueSuffix, ArgSeparator: tf.ArgSeparator, ArgsStart: tf.ArgsStart, ArgsEnd: tf.ArgsEnd, NameField: tf.NameField, ArgsField: tf.ArgsField, IDField: tf.IdField, FunNameIsKey: tf.FunNameIsKey, ToolsArrayWrapped: tf.ToolsArrayWrapped, FunctionField: tf.FunctionField, ParameterOrder: tf.ParameterOrder, GenIDField: tf.GenIdField, CallIDPosition: tf.CallIdPosition, CallIDPrefix: tf.CallIdPrefix, CallIDSuffix: tf.CallIdSuffix, ReasoningStart: tf.ReasoningStart, ReasoningEnd: tf.ReasoningEnd, ContentStart: tf.ContentStart, ContentEnd: tf.ContentEnd, } xlog.Debug("[gguf] DetectThinkingSupportFromBackend: tool format markers detected", "format_type", tf.FormatType, "section_start", tf.SectionStart, "func_name_prefix", tf.FuncNamePrefix) } } } func applyDetectedThinkingConfig(cfg *ModelConfig, metadata *pb.ModelMetadataResponse) { if cfg == nil || metadata == nil { return } // Respect explicit YAML/user config. Backend probing should only fill defaults // when the reasoning mode has not already been set. if cfg.ReasoningConfig.DisableReasoning == nil { cfg.ReasoningConfig.DisableReasoning = ptr.To(!metadata.SupportsThinking) } // Respect explicit prefill config for the same reason. Only infer the // default prefill behavior when the user did not set it. if cfg.ReasoningConfig.DisableReasoningTagPrefill == nil { // Use the rendered template to detect if thinking token is at the end. // This reuses the existing DetectThinkingStartToken function. if metadata.RenderedTemplate != "" { thinkingStartToken := reasoning.DetectThinkingStartToken(metadata.RenderedTemplate, &cfg.ReasoningConfig) thinkingForcedOpen := thinkingStartToken != "" cfg.ReasoningConfig.DisableReasoningTagPrefill = ptr.To(!thinkingForcedOpen) xlog.Debug("[gguf] DetectThinkingSupportFromBackend: thinking support detected", "supports_thinking", metadata.SupportsThinking, "thinking_forced_open", thinkingForcedOpen, "thinking_start_token", thinkingStartToken) } else { cfg.ReasoningConfig.DisableReasoningTagPrefill = ptr.To(true) xlog.Debug("[gguf] DetectThinkingSupportFromBackend: thinking support detected", "supports_thinking", metadata.SupportsThinking, "thinking_forced_open", false) } return } xlog.Debug("[gguf] DetectThinkingSupportFromBackend: preserving explicit reasoning config", "supports_thinking", metadata.SupportsThinking, "disable_reasoning", *cfg.ReasoningConfig.DisableReasoning, "disable_reasoning_tag_prefill", *cfg.ReasoningConfig.DisableReasoningTagPrefill) }