mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-29 01:28:43 -04:00
mudler-agentandEttore Di Giacinto
8a03ce7f6c
Aliases have no model file, so guessing attempts to parse the models directory. Skip that work while retaining the context fallback and normal alias validation. Use a private call-local parser dependency to verify operation counts without a mutable global hook. Cover concrete guessing, defaults, alias validation, and resolved target equivalence. Assisted-by: nib:gpt-6-astra Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
72 lines
2.4 KiB
Go
72 lines
2.4 KiB
Go
package config
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
|
|
gguf "github.com/gpustack/gguf-parser-go"
|
|
"github.com/mudler/xlog"
|
|
)
|
|
|
|
func init() {
|
|
// Register for both explicit llama-cpp and empty backend (auto-detect from GGUF file)
|
|
RegisterBackendHook("llama-cpp", llamaCppDefaults)
|
|
RegisterBackendHook("", llamaCppDefaults)
|
|
}
|
|
|
|
func llamaCppDefaults(cfg *ModelConfig, modelPath string) {
|
|
llamaCppDefaultsWithParser(cfg, modelPath, gguf.ParseGGUFFile)
|
|
}
|
|
|
|
func llamaCppDefaultsWithParser(cfg *ModelConfig, modelPath string, parse func(string, ...gguf.GGUFReadOption) (*gguf.GGUFFile, error)) {
|
|
if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
|
|
xlog.Debug("llamaCppDefaults: guessing disabled")
|
|
return
|
|
}
|
|
if modelPath == "" {
|
|
return
|
|
}
|
|
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
xlog.Error("llamaCppDefaults: panic while parsing gguf file")
|
|
}
|
|
}()
|
|
|
|
// Default context size if not set, or if a context_size=-1 auto-max was
|
|
// requested but the GGUF could not be parsed, so guessGGUFFromFile never ran
|
|
// to resolve it. A negative value must never reach the backend.
|
|
defer func() {
|
|
if cfg.ContextSize == nil || *cfg.ContextSize < 0 {
|
|
ctx := DefaultContextSize
|
|
cfg.ContextSize = &ctx
|
|
}
|
|
}()
|
|
|
|
// Aliases have no model file, but still need the context fallback above.
|
|
if cfg.IsAlias() {
|
|
return
|
|
}
|
|
|
|
guessPath := filepath.Join(modelPath, cfg.ModelFileName())
|
|
|
|
// Startup parses every model's GGUF header to guess defaults. We only need
|
|
// scalar metadata (architecture, head/ff counts, chat_template, token IDs,
|
|
// MTP head) plus array *lengths* — never the array *contents*. Two options
|
|
// keep this cheap, which matters when many models live on slow storage such
|
|
// as a Docker volume (see https://github.com/mudler/LocalAI/issues/9790):
|
|
//
|
|
// - SkipLargeMetadata: seek past large array-valued metadata (the tokenizer
|
|
// vocab: tokenizer.ggml.tokens/scores/merges, often >100k entries) instead
|
|
// of reading and allocating every element. Lengths stay populated.
|
|
// - UseMMap: read the header via a memory map so faulting in a few pages
|
|
// replaces hundreds of thousands of tiny read() syscalls (measured ~524k
|
|
// -> 8 for a 256k-token vocab), the dominant cost on slow filesystems.
|
|
//
|
|
// The mapping is released when ParseGGUFFile returns.
|
|
f, err := parse(guessPath, gguf.UseMMap(), gguf.SkipLargeMetadata())
|
|
if err == nil {
|
|
guessGGUFFromFile(cfg, f, 0)
|
|
}
|
|
}
|