From 5569b2de5688a25ac8751fdd9c598a4e098cddeb Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:03:40 +0200 Subject: [PATCH] feat(config): context_size: -1 to auto-use model's full trained context (#10752) * feat(config): clamp negative context_size to default in EffectiveContextSize Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-4-8 [Claude Code] * feat(config): resolve context_size=-1 to model trained max with VRAM warn Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-4-8 [Claude Code] * fix(config): treat negative context_size as unset when GGUF is unparseable Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-4-8 [Claude Code] * docs(config): document context_size=-1 auto-max Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-4-8 [Claude Code] * docs(backend): drop em dashes from EffectiveContextSize comment Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-4-8 [Claude Code] --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/backend/options.go | 7 ++- core/backend/options_internal_test.go | 21 ++++++++ core/config/gguf.go | 50 ++++++++++++++++++++ core/config/hooks_llamacpp.go | 6 ++- core/config/hooks_test.go | 44 +++++++++++++++-- docs/content/advanced/model-configuration.md | 2 +- docs/content/reference/cli-reference.md | 2 +- 7 files changed, 123 insertions(+), 9 deletions(-) diff --git a/core/backend/options.go b/core/backend/options.go index 8fc47f231..29b84d924 100644 --- a/core/backend/options.go +++ b/core/backend/options.go @@ -215,9 +215,12 @@ const ( ) // EffectiveContextSize is the context window the backend will run with: the -// configured value, or DefaultContextSize when unset. +// configured value, or DefaultContextSize when unset. A negative value (the +// context_size: -1 auto-max sentinel) that survived config resolution, e.g. on +// a backend that never ran the GGUF resolver, is clamped here so a negative +// n_ctx never reaches a backend. func EffectiveContextSize(c config.ModelConfig) int { - if c.ContextSize != nil { + if c.ContextSize != nil && *c.ContextSize > 0 { return *c.ContextSize } return DefaultContextSize diff --git a/core/backend/options_internal_test.go b/core/backend/options_internal_test.go index 723adb4de..f8ac7244a 100644 --- a/core/backend/options_internal_test.go +++ b/core/backend/options_internal_test.go @@ -293,3 +293,24 @@ var _ = Describe("gRPCPredictOpts chat_template_kwargs metadata", func() { Expect(opts.Metadata).ToNot(HaveKey("chat_template_kwargs")) }) }) + +var _ = Describe("EffectiveContextSize", func() { + Context("EffectiveContextSize", func() { + It("clamps a negative (auto-max sentinel) context size to the default", func() { + neg := -1 + cfg := config.ModelConfig{LLMConfig: config.LLMConfig{ContextSize: &neg}} + Expect(EffectiveContextSize(cfg)).To(Equal(DefaultContextSize)) + }) + + It("returns an explicit positive context size unchanged", func() { + ctx := 8192 + cfg := config.ModelConfig{LLMConfig: config.LLMConfig{ContextSize: &ctx}} + Expect(EffectiveContextSize(cfg)).To(Equal(8192)) + }) + + It("falls back to the default when context size is unset", func() { + cfg := config.ModelConfig{} + Expect(EffectiveContextSize(cfg)).To(Equal(DefaultContextSize)) + }) + }) +}) diff --git a/core/config/gguf.go b/core/config/gguf.go index 6e6c796e1..186c9438f 100644 --- a/core/config/gguf.go +++ b/core/config/gguf.go @@ -27,6 +27,24 @@ func reservedNonChatModel(cfg *ModelConfig) bool { } func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) { + // Explicit opt-in: a negative context_size (canonically -1) means "use the + // model's full trained context (n_ctx_train) from GGUF metadata". Unlike the + // silent unset path below, this overrides an already-present value and warns + // when the resolved window will not fit detected VRAM. + if cfg.ContextSize != nil && *cfg.ContextSize < 0 { + if maxCtx := int(f.Architecture().MaximumContextLength); maxCtx > 0 { + cfg.ContextSize = &maxCtx + warnIfContextExceedsVRAM(f, maxCtx, f.Metadata().Name) + } else { + // No usable trained max in metadata: degrade to the safe default + // rather than leak a negative n_ctx downstream. + d := DefaultContextSize + cfg.ContextSize = &d + xlog.Warn("[gguf] context_size=-1 requested but GGUF exposes no trained max; using default", + "default", d, "model", f.Metadata().Name) + } + } + if defaultCtx == 0 && cfg.ContextSize == nil { // trainedMax is the model's full trained context window (n_ctx_train). // Defaulting a model to it unbounded is what OOMs long-context models at @@ -225,3 +243,35 @@ func applyDetectedThinkingConfig(cfg *ModelConfig, metadata *pb.ModelMetadataRes xlog.Debug("[gguf] DetectThinkingSupportFromBackend: preserving explicit reasoning config", "supports_thinking", metadata.SupportsThinking, "disable_reasoning", *cfg.ReasoningConfig.DisableReasoning, "disable_reasoning_tag_prefill", *cfg.ReasoningConfig.DisableReasoningTagPrefill) } + +// warnIfContextExceedsVRAM logs a best-effort warning when running the model at +// the given context would not fit detected VRAM. It never blocks load: any +// detection or estimation gap (no GPU, unknown VRAM, estimate failure) silently +// skips the warning. Used by the context_size=-1 auto-max path, where the raw +// trained max can be far larger than a consumer card holds. +func warnIfContextExceedsVRAM(f *gguf.GGUFFile, ctx int, name string) { + defer func() { _ = recover() }() // the run estimate can panic on unusual headers + + if !xsysinfo.HasGPU("nvidia") && !xsysinfo.HasGPU("amd") { + return // no VRAM to compare against + } + vram, err := xsysinfo.TotalAvailableVRAM() + if err != nil || vram == 0 { + return + } + + sum := f.EstimateLLaMACppRun(gguf.WithLLaMACppContextSize(int32(ctx))).Summarize(true, 0, 0) + if len(sum.Items) == 0 { + return + } + var used uint64 + for _, v := range sum.Items[0].VRAMs { + used += uint64(v.NonUMA) + } + if used == 0 || used <= vram { + return + } + xlog.Warn("[gguf] context_size=-1 resolved to the model's trained max; estimated VRAM may exceed available - expect OOM, or set an explicit context_size", + "model", name, "context", ctx, + "estimated_vram_gib", used>>30, "available_vram_gib", vram>>30) +} diff --git a/core/config/hooks_llamacpp.go b/core/config/hooks_llamacpp.go index 07ccdda7b..d4e77a205 100644 --- a/core/config/hooks_llamacpp.go +++ b/core/config/hooks_llamacpp.go @@ -31,9 +31,11 @@ func llamaCppDefaults(cfg *ModelConfig, modelPath string) { } }() - // Default context size if not set, regardless of whether GGUF parsing succeeds + // Default context size if not set, or if a context_size=-1 auto-max was + // requested but the GGUF could not be parsed, so guessGGUFFromFile never ran + // to resolve it. A negative value must never reach the backend. defer func() { - if cfg.ContextSize == nil { + if cfg.ContextSize == nil || *cfg.ContextSize < 0 { ctx := DefaultContextSize cfg.ContextSize = &ctx } diff --git a/core/config/hooks_test.go b/core/config/hooks_test.go index a1b30b8d9..4a6e31d6c 100644 --- a/core/config/hooks_test.go +++ b/core/config/hooks_test.go @@ -26,7 +26,7 @@ const ( // array (tokenizer.ggml.tokens). The big array is exactly what SkipLargeMetadata // + UseMMap are expected to avoid reading element-by-element, so it must survive a // round-trip through the real hook without corrupting the guessed defaults. -func writeTestGGUF(path, chatTemplate string, vocab int) error { +func writeTestGGUF(path, chatTemplate string, vocab int, ctxTrain uint32) error { wStr := func(b *bytes.Buffer, s string) { binary.Write(b, binary.LittleEndian, uint64(len(s))) b.WriteString(s) @@ -45,7 +45,7 @@ func writeTestGGUF(path, chatTemplate string, vocab int) error { var meta bytes.Buffer kvStr(&meta, "general.architecture", "llama") kvStr(&meta, "general.name", "ReproModel") - kvU32(&meta, "llama.context_length", 4096) + kvU32(&meta, "llama.context_length", ctxTrain) kvU32(&meta, "llama.attention.head_count", 32) kvU32(&meta, "llama.feed_forward_length", 11008) kvU32(&meta, "llama.block_count", 32) @@ -211,7 +211,7 @@ var _ = Describe("Backend hooks and parser defaults", func() { It("guesses defaults from a GGUF whose large vocab is skipped", func() { dir := GinkgoT().TempDir() modelFile := "repro.gguf" - Expect(writeTestGGUF(filepath.Join(dir, modelFile), chatTemplate, 50000)).To(Succeed()) + Expect(writeTestGGUF(filepath.Join(dir, modelFile), chatTemplate, 50000, 4096)).To(Succeed()) // A pre-set context size short-circuits the GGUF run-estimate, which // needs full tensor info this header-only fixture deliberately omits; @@ -236,6 +236,26 @@ var _ = Describe("Backend hooks and parser defaults", func() { Expect(cfg.KnownUsecaseStrings).To(ContainElement("FLAG_CHAT")) }) + It("resolves context_size=-1 to the model's trained maximum context", func() { + dir := GinkgoT().TempDir() + modelFile := "automax.gguf" + // A distinctive trained max proves we read metadata, not the 4096 default. + Expect(writeTestGGUF(filepath.Join(dir, modelFile), chatTemplate, 100, 131072)).To(Succeed()) + + neg := -1 + cfg := &ModelConfig{ + Backend: "llama-cpp", + LLMConfig: LLMConfig{ContextSize: &neg}, + PredictionOptions: schema.PredictionOptions{ + BasicModelRequest: schema.BasicModelRequest{Model: modelFile}, + }, + } + cfg.SetDefaults(ModelPath(dir)) + + Expect(cfg.ContextSize).NotTo(BeNil()) + Expect(*cfg.ContextSize).To(Equal(131072)) + }) + It("falls back to the default context size when the GGUF is unreadable", func() { dir := GinkgoT().TempDir() Expect(os.WriteFile(filepath.Join(dir, "bad.gguf"), []byte("not a gguf"), 0o644)).To(Succeed()) @@ -254,6 +274,24 @@ var _ = Describe("Backend hooks and parser defaults", func() { Expect(cfg.ContextSize).NotTo(BeNil()) Expect(*cfg.ContextSize).To(Equal(DefaultContextSize)) }) + + It("falls back to the default when context_size=-1 but the GGUF is unreadable", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, "bad.gguf"), []byte("not a gguf"), 0o644)).To(Succeed()) + + neg := -1 + cfg := &ModelConfig{ + Backend: "llama-cpp", + LLMConfig: LLMConfig{ContextSize: &neg}, + PredictionOptions: schema.PredictionOptions{ + BasicModelRequest: schema.BasicModelRequest{Model: "bad.gguf"}, + }, + } + cfg.SetDefaults(ModelPath(dir)) + + Expect(cfg.ContextSize).NotTo(BeNil()) + Expect(*cfg.ContextSize).To(Equal(DefaultContextSize)) + }) }) Context("PromptCacheAll default", func() { diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index 2fcb84c35..b5ec727e6 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -145,7 +145,7 @@ These settings apply to most LLM backends (llama.cpp, vLLM, etc.): | Field | Type | Default | Description | |-------|------|---------|-------------| | `threads` | int | `processor count` | Number of threads for parallel computation | -| `context_size` | int | `512` | Maximum context size (number of tokens) | +| `context_size` | int | `512` | Maximum context size in tokens. Set to `-1` to auto-use the model's full trained context from GGUF metadata (raw max, no VRAM capping; a warning is logged if it may not fit detected VRAM). | | `f16` | bool | `false` | Enable 16-bit floating point precision (GPU acceleration) | | `gpu_layers` | int | `0` | Number of layers to offload to GPU (0 = CPU only) | diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index 50ba18705..0146192aa 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -73,7 +73,7 @@ For more information on VRAM management, see [VRAM and Memory Management]({{%rel |-----------|---------|-------------|----------------------| | `--f16` | `false` | Enable GPU acceleration | `$LOCALAI_F16`, `$F16` | | `-t, --threads` | | Number of threads used for parallel computation. Usage of the number of physical cores in the system is suggested | `$LOCALAI_THREADS`, `$THREADS` | -| `--context-size` | | Default context size for models | `$LOCALAI_CONTEXT_SIZE`, `$CONTEXT_SIZE` | +| `--context-size` | | Default context size for models (`-1` = each model's full trained context from GGUF metadata) | `$LOCALAI_CONTEXT_SIZE`, `$CONTEXT_SIZE` | ## API Flags