diff --git a/core/config/hooks_llamacpp.go b/core/config/hooks_llamacpp.go index d4e77a205..181d43330 100644 --- a/core/config/hooks_llamacpp.go +++ b/core/config/hooks_llamacpp.go @@ -15,6 +15,10 @@ func init() { } func llamaCppDefaults(cfg *ModelConfig, modelPath string) { + llamaCppDefaultsWithParser(cfg, modelPath, gguf.ParseGGUFFile) +} + +func llamaCppDefaultsWithParser(cfg *ModelConfig, modelPath string, parse func(string, ...gguf.GGUFReadOption) (*gguf.GGUFFile, error)) { if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" { xlog.Debug("llamaCppDefaults: guessing disabled") return @@ -23,8 +27,6 @@ func llamaCppDefaults(cfg *ModelConfig, modelPath string) { return } - guessPath := filepath.Join(modelPath, cfg.ModelFileName()) - defer func() { if r := recover(); r != nil { xlog.Error("llamaCppDefaults: panic while parsing gguf file") @@ -41,6 +43,13 @@ func llamaCppDefaults(cfg *ModelConfig, modelPath string) { } }() + // Aliases have no model file, but still need the context fallback above. + if cfg.IsAlias() { + return + } + + guessPath := filepath.Join(modelPath, cfg.ModelFileName()) + // Startup parses every model's GGUF header to guess defaults. We only need // scalar metadata (architecture, head/ff counts, chat_template, token IDs, // MTP head) plus array *lengths* — never the array *contents*. Two options @@ -55,7 +64,7 @@ func llamaCppDefaults(cfg *ModelConfig, modelPath string) { // -> 8 for a 256k-token vocab), the dominant cost on slow filesystems. // // The mapping is released when ParseGGUFFile returns. - f, err := gguf.ParseGGUFFile(guessPath, gguf.UseMMap(), gguf.SkipLargeMetadata()) + f, err := parse(guessPath, gguf.UseMMap(), gguf.SkipLargeMetadata()) if err == nil { guessGGUFFromFile(cfg, f, 0) } diff --git a/core/config/hooks_llamacpp_test.go b/core/config/hooks_llamacpp_test.go new file mode 100644 index 000000000..c33dcc6d6 --- /dev/null +++ b/core/config/hooks_llamacpp_test.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT +package config + +import ( + "errors" + "os" + "path/filepath" + "reflect" + + gguf "github.com/gpustack/gguf-parser-go" + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("AliasGuessing", func() { + BeforeEach(func() { GinkgoT().Setenv("LOCALAI_DISABLE_GUESSING", "false") }) + + DescribeTable("skips the parser for aliases while preserving context defaults", + func(context *int, expected int) { + cfg := &ModelConfig{Name: "production", Alias: "target", LLMConfig: LLMConfig{ContextSize: context}} + calls := 0 + llamaCppDefaultsWithParser(cfg, GinkgoT().TempDir(), func(path string, opts ...gguf.GGUFReadOption) (*gguf.GGUFFile, error) { + calls++ + return gguf.ParseGGUFFile(path, opts...) + }) + Expect(calls).To(Equal(0), "an alias must not parse the models directory") + Expect(cfg.ContextSize).To(HaveValue(Equal(expected))) + }, + Entry("unset context", nil, DefaultContextSize), + Entry("auto-max context", aliasGuessingInt(-1), DefaultContextSize), + Entry("custom context", aliasGuessingInt(8192), 8192), + Entry("zero context", aliasGuessingInt(0), 0), + ) + + DescribeTable("keeps file-backed guessing and parser options", + func(backend string) { + dir := GinkgoT().TempDir() + cfg := &ModelConfig{Backend: backend, PredictionOptions: schema.PredictionOptions{BasicModelRequest: schema.BasicModelRequest{Model: "weights.gguf"}}} + calls := 0 + var gotPath string + var gotOptions []gguf.GGUFReadOption + llamaCppDefaultsWithParser(cfg, dir, func(path string, opts ...gguf.GGUFReadOption) (*gguf.GGUFFile, error) { + calls++ + gotPath, gotOptions = path, opts + return nil, errors.New("unreadable fixture") + }) + Expect(calls).To(Equal(1)) + Expect(gotPath).To(Equal(filepath.Join(dir, "weights.gguf"))) + Expect(gotOptions).To(HaveLen(2)) + // The parser's option struct is private; inspect the applied public options. + options := reflect.New(reflect.TypeOf(gotOptions[0]).In(0).Elem()) + for _, opt := range gotOptions { + reflect.ValueOf(opt).Call([]reflect.Value{options}) + } + Expect(options.Elem().FieldByName("MMap").Bool()).To(BeTrue()) + Expect(options.Elem().FieldByName("SkipLargeMetadata").Bool()).To(BeTrue()) + Expect(cfg.ContextSize).To(HaveValue(Equal(DefaultContextSize))) + }, + Entry("explicit llama-cpp", "llama-cpp"), + Entry("auto-detected backend", ""), + ) + + It("keeps the global disable and empty model path behavior", func() { + for _, disabled := range []string{"false", "true"} { + GinkgoT().Setenv("LOCALAI_DISABLE_GUESSING", disabled) + path := "" + if disabled == "true" { + path = GinkgoT().TempDir() + } + cfg := &ModelConfig{Backend: "llama-cpp"} + calls := 0 + llamaCppDefaultsWithParser(cfg, path, func(string, ...gguf.GGUFReadOption) (*gguf.GGUFFile, error) { + calls++ + return nil, errors.New("unexpected parse") + }) + Expect(calls).To(BeZero()) + Expect(cfg.ContextSize).To(BeNil()) + } + }) + + It("retains generic and loader defaults through SetDefaults", func() { + cfg := &ModelConfig{Name: "production", Alias: "target"} + cfg.SetDefaults(ModelPath(GinkgoT().TempDir()), LoadOptionThreads(7), LoadOptionContextSize(8192)) + Expect(cfg.ContextSize).To(HaveValue(Equal(8192))) + Expect(cfg.Threads).To(HaveValue(Equal(7))) + Expect(cfg.PromptCacheAll).To(HaveValue(BeTrue())) + valid, err := cfg.Validate() + Expect(err).NotTo(HaveOccurred()) + Expect(valid).To(BeTrue()) + }) + + DescribeTable("does not accept malformed aliases after SetDefaults", + func(cfg ModelConfig, message string) { + cfg.SetDefaults(ModelPath(GinkgoT().TempDir())) + valid, err := cfg.Validate() + Expect(valid).To(BeFalse()) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("missing name", ModelConfig{Alias: "target"}, "requires a name"), + Entry("self reference", ModelConfig{Name: "loop", Alias: "loop"}, "itself"), + Entry("backend", ModelConfig{Name: "production", Alias: "target", Backend: "llama-cpp"}, "pure redirect"), + Entry("model file", ModelConfig{Name: "production", Alias: "target", PredictionOptions: schema.PredictionOptions{BasicModelRequest: schema.BasicModelRequest{Model: "weights.gguf"}}}, "pure redirect"), + ) + + It("resolves to the same effective target config without leaking alias defaults", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, "target.yaml"), []byte("name: target\nbackend: llama-cpp\ncontext_size: 16384\nthreads: 3\nparameters:\n model: weights.gguf\n"), 0600)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "alias.yaml"), []byte("name: production\nalias: target\ncontext_size: 8192\nthreads: 9\n"), 0600)).To(Succeed()) + loader := NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir)).To(Succeed()) + resolved, err := loader.LoadResolvedModelConfig("production", dir) + Expect(err).NotTo(HaveOccurred()) + direct, err := loader.LoadResolvedModelConfig("target", dir) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved).To(Equal(direct)) + Expect(resolved.IsAlias()).To(BeFalse()) + Expect(resolved.ContextSize).To(HaveValue(Equal(16384))) + Expect(resolved.Threads).To(HaveValue(Equal(3))) + }) +}) + +func aliasGuessingInt(value int) *int { return &value }