mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 18:09:05 -04:00
b224c96db6
SetDefaults injected the llama.cpp server options cache_reuse (ApplyServingDefaults) and parallel (ApplyHardwareDefaults, re-applied per selected node by the distributed router) onto every model config regardless of backend. Every other backend ignores options it does not understand, so this was harmless until longcat-video, which strictly validates its options and fails LoadModel with "unknown model option(s): cache_reuse, parallel". Gate both injections behind a new UsesLlamaCppServingOptions allow-list (llama-cpp plus the empty/auto-detect case that resolves to llama.cpp from a GGUF file, mirroring how llamaCppDefaults is registered). This follows the existing UsesLlamaSamplerDefaults precedent for llama-only defaults. The typed NBatch field is deliberately left alone: it is a proto field every backend simply ignores, which is why batch never triggered the error. Also harden the longcat-video backend to warn-and-ignore unknown model options and request params through a testable select_known_options helper, matching the other LocalAI Python backends, so a future server-injected option cannot break loading again. Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
46 lines
1.6 KiB
Go
46 lines
1.6 KiB
Go
package config_test
|
|
|
|
import (
|
|
. "github.com/mudler/LocalAI/core/config"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = Describe("Serving-policy config defaults", func() {
|
|
Describe("ApplyServingDefaults (cross-request prefix cache)", func() {
|
|
It("enables cache_reuse when unset", func() {
|
|
cfg := &ModelConfig{}
|
|
ApplyServingDefaults(cfg)
|
|
Expect(cfg.Options).To(ContainElement("cache_reuse:256"))
|
|
})
|
|
It("never overrides an explicit cache_reuse", func() {
|
|
cfg := &ModelConfig{Options: []string{"cache_reuse:0"}}
|
|
ApplyServingDefaults(cfg)
|
|
Expect(cfg.Options).To(Equal([]string{"cache_reuse:0"}))
|
|
})
|
|
It("recognizes the n_cache_reuse alias", func() {
|
|
cfg := &ModelConfig{Options: []string{"n_cache_reuse:512"}}
|
|
ApplyServingDefaults(cfg)
|
|
Expect(cfg.Options).To(Equal([]string{"n_cache_reuse:512"}))
|
|
})
|
|
It("no-ops on nil", func() {
|
|
Expect(func() { ApplyServingDefaults(nil) }).ToNot(Panic())
|
|
})
|
|
It("does not enable cache_reuse for a non-llama backend", func() {
|
|
// cache_reuse is a llama.cpp server option (n_cache_reuse). Backends
|
|
// that strictly validate their options (e.g. longcat-video) reject it
|
|
// with "unknown model option(s)". Only the llama.cpp path gets it.
|
|
cfg := &ModelConfig{}
|
|
cfg.Backend = "longcat-video"
|
|
ApplyServingDefaults(cfg)
|
|
Expect(cfg.Options).ToNot(ContainElement(ContainSubstring("cache_reuse")))
|
|
})
|
|
It("still enables cache_reuse for an explicit llama-cpp backend", func() {
|
|
cfg := &ModelConfig{}
|
|
cfg.Backend = "llama-cpp"
|
|
ApplyServingDefaults(cfg)
|
|
Expect(cfg.Options).To(ContainElement("cache_reuse:256"))
|
|
})
|
|
})
|
|
})
|