LocalAI/core/config/inference_defaults.go

package config

//go:generate go run ./gen_inference_defaults/

import (
	_ "embed"
	"encoding/json"
	"strings"

	"github.com/mudler/xlog"
)

//go:embed inference_defaults.json
var inferenceDefaultsJSON []byte

// inferenceDefaults holds the parsed inference defaults data
type inferenceDefaults struct {
	Families map[string]map[string]float64 `json:"families"`
	Patterns []string                      `json:"patterns"`
}

var defaultsData *inferenceDefaults

func init() {
	defaultsData = &inferenceDefaults{}
	if err := json.Unmarshal(inferenceDefaultsJSON, defaultsData); err != nil {
		xlog.Warn("failed to parse inference_defaults.json", "error", err)
	}
}

// normalizeModelID lowercases, strips org prefix (before /), and removes .gguf extension
func normalizeModelID(modelID string) string {
	modelID = strings.ToLower(modelID)

	// Strip org prefix (e.g., "unsloth/Qwen3.5-9B-GGUF" -> "qwen3.5-9b-gguf")
	if idx := strings.LastIndex(modelID, "/"); idx >= 0 {
		modelID = modelID[idx+1:]
	}

	// Strip .gguf extension
	modelID = strings.TrimSuffix(modelID, ".gguf")

	// Replace underscores with hyphens for matching
	modelID = strings.ReplaceAll(modelID, "_", "-")

	return modelID
}

// MatchModelFamily returns the inference defaults for the best-matching model family.
// Patterns are checked in order (longest-match-first as defined in the JSON).
// Returns nil if no family matches.
func MatchModelFamily(modelID string) map[string]float64 {
	if defaultsData == nil || len(defaultsData.Patterns) == 0 {
		return nil
	}

	normalized := normalizeModelID(modelID)

	for _, pattern := range defaultsData.Patterns {
		if strings.Contains(normalized, pattern) {
			if family, ok := defaultsData.Families[pattern]; ok {
				return family
			}
		}
	}

	return nil
}

// ApplyInferenceDefaults sets recommended inference parameters on cfg based on modelIDs.
// Tries each modelID in order; the first match wins.
// Only fills in parameters that are not already set (nil pointers or zero values).
func ApplyInferenceDefaults(cfg *ModelConfig, modelIDs ...string) {
	var family map[string]float64
	var matchedID string
	for _, id := range modelIDs {
		if id == "" {
			continue
		}
		if f := MatchModelFamily(id); f != nil {
			family = f
			matchedID = id
			break
		}
	}
	if family == nil {
		return
	}

	xlog.Debug("[inference_defaults] applying defaults for model", "modelID", matchedID, "family", family)

	if cfg.Temperature == nil {
		if v, ok := family["temperature"]; ok {
			cfg.Temperature = &v
		}
	}

	if cfg.TopP == nil {
		if v, ok := family["top_p"]; ok {
			cfg.TopP = &v
		}
	}

	if cfg.TopK == nil {
		if v, ok := family["top_k"]; ok {
			intV := int(v)
			cfg.TopK = &intV
		}
	}

	if cfg.MinP == nil {
		if v, ok := family["min_p"]; ok {
			cfg.MinP = &v
		}
	}

	if cfg.RepeatPenalty == 0 {
		if v, ok := family["repeat_penalty"]; ok {
			cfg.RepeatPenalty = v
		}
	}

	if cfg.PresencePenalty == 0 {
		if v, ok := family["presence_penalty"]; ok {
			cfg.PresencePenalty = v
		}
	}
}