feat(config): context_size: -1 to auto-use model's full trained context (#10752)

* feat(config): clamp negative context_size to default in EffectiveContextSize Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude:claude-opus-4-8 [Claude Code] * feat(config): resolve context_size=-1 to model trained max with VRAM warn Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude:claude-opus-4-8 [Claude Code] * fix(config): treat negative context_size as unset when GGUF is unparseable Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude:claude-opus-4-8 [Claude Code] * docs(config): document context_size=-1 auto-max Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude:claude-opus-4-8 [Claude Code] * docs(backend): drop em dashes from EffectiveContextSize comment Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude:claude-opus-4-8 [Claude Code] --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
2026-07-10 08:18:33 -04:00 · 2026-07-09 09:03:40 +02:00
parent c9f73f40ff
commit 5569b2de56
7 changed files with 123 additions and 9 deletions
--- a/core/backend/options.go
+++ b/core/backend/options.go
@@ -215,9 +215,12 @@ const (
 )

 // EffectiveContextSize is the context window the backend will run with: the
-// configured value, or DefaultContextSize when unset.
+// configured value, or DefaultContextSize when unset. A negative value (the
+// context_size: -1 auto-max sentinel) that survived config resolution, e.g. on
+// a backend that never ran the GGUF resolver, is clamped here so a negative
+// n_ctx never reaches a backend.
 func EffectiveContextSize(c config.ModelConfig) int {
-	if c.ContextSize != nil {
+	if c.ContextSize != nil && *c.ContextSize > 0 {
 		return *c.ContextSize
 	}
 	return DefaultContextSize
--- a/core/backend/options_internal_test.go
+++ b/core/backend/options_internal_test.go
@@ -293,3 +293,24 @@ var _ = Describe("gRPCPredictOpts chat_template_kwargs metadata", func() {
 		Expect(opts.Metadata).ToNot(HaveKey("chat_template_kwargs"))
 	})
 })
+
+var _ = Describe("EffectiveContextSize", func() {
+	Context("EffectiveContextSize", func() {
+		It("clamps a negative (auto-max sentinel) context size to the default", func() {
+			neg := -1
+			cfg := config.ModelConfig{LLMConfig: config.LLMConfig{ContextSize: &neg}}
+			Expect(EffectiveContextSize(cfg)).To(Equal(DefaultContextSize))
+		})
+
+		It("returns an explicit positive context size unchanged", func() {
+			ctx := 8192
+			cfg := config.ModelConfig{LLMConfig: config.LLMConfig{ContextSize: &ctx}}
+			Expect(EffectiveContextSize(cfg)).To(Equal(8192))
+		})
+
+		It("falls back to the default when context size is unset", func() {
+			cfg := config.ModelConfig{}
+			Expect(EffectiveContextSize(cfg)).To(Equal(DefaultContextSize))
+		})
+	})
+})
--- a/core/config/gguf.go
+++ b/core/config/gguf.go
@@ -27,6 +27,24 @@ func reservedNonChatModel(cfg *ModelConfig) bool {
 }

 func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
+	// Explicit opt-in: a negative context_size (canonically -1) means "use the
+	// model's full trained context (n_ctx_train) from GGUF metadata". Unlike the
+	// silent unset path below, this overrides an already-present value and warns
+	// when the resolved window will not fit detected VRAM.
+	if cfg.ContextSize != nil && *cfg.ContextSize < 0 {
+		if maxCtx := int(f.Architecture().MaximumContextLength); maxCtx > 0 {
+			cfg.ContextSize = &maxCtx
+			warnIfContextExceedsVRAM(f, maxCtx, f.Metadata().Name)
+		} else {
+			// No usable trained max in metadata: degrade to the safe default
+			// rather than leak a negative n_ctx downstream.
+			d := DefaultContextSize
+			cfg.ContextSize = &d
+			xlog.Warn("[gguf] context_size=-1 requested but GGUF exposes no trained max; using default",
+				"default", d, "model", f.Metadata().Name)
+		}
+	}
+
 	if defaultCtx == 0 && cfg.ContextSize == nil {
 		// trainedMax is the model's full trained context window (n_ctx_train).
 		// Defaulting a model to it unbounded is what OOMs long-context models at
@@ -225,3 +243,35 @@ func applyDetectedThinkingConfig(cfg *ModelConfig, metadata *pb.ModelMetadataRes

 	xlog.Debug("[gguf] DetectThinkingSupportFromBackend: preserving explicit reasoning config", "supports_thinking", metadata.SupportsThinking, "disable_reasoning", *cfg.ReasoningConfig.DisableReasoning, "disable_reasoning_tag_prefill", *cfg.ReasoningConfig.DisableReasoningTagPrefill)
 }
+
+// warnIfContextExceedsVRAM logs a best-effort warning when running the model at
+// the given context would not fit detected VRAM. It never blocks load: any
+// detection or estimation gap (no GPU, unknown VRAM, estimate failure) silently
+// skips the warning. Used by the context_size=-1 auto-max path, where the raw
+// trained max can be far larger than a consumer card holds.
+func warnIfContextExceedsVRAM(f *gguf.GGUFFile, ctx int, name string) {
+	defer func() { _ = recover() }() // the run estimate can panic on unusual headers
+
+	if !xsysinfo.HasGPU("nvidia") && !xsysinfo.HasGPU("amd") {
+		return // no VRAM to compare against
+	}
+	vram, err := xsysinfo.TotalAvailableVRAM()
+	if err != nil || vram == 0 {
+		return
+	}
+
+	sum := f.EstimateLLaMACppRun(gguf.WithLLaMACppContextSize(int32(ctx))).Summarize(true, 0, 0)
+	if len(sum.Items) == 0 {
+		return
+	}
+	var used uint64
+	for _, v := range sum.Items[0].VRAMs {
+		used += uint64(v.NonUMA)
+	}
+	if used == 0 || used <= vram {
+		return
+	}
+	xlog.Warn("[gguf] context_size=-1 resolved to the model's trained max; estimated VRAM may exceed available - expect OOM, or set an explicit context_size",
+		"model", name, "context", ctx,
+		"estimated_vram_gib", used>>30, "available_vram_gib", vram>>30)
+}
--- a/core/config/hooks_llamacpp.go
+++ b/core/config/hooks_llamacpp.go
@@ -31,9 +31,11 @@ func llamaCppDefaults(cfg *ModelConfig, modelPath string) {
 		}
 	}()

-	// Default context size if not set, regardless of whether GGUF parsing succeeds
+	// Default context size if not set, or if a context_size=-1 auto-max was
+	// requested but the GGUF could not be parsed, so guessGGUFFromFile never ran
+	// to resolve it. A negative value must never reach the backend.
 	defer func() {
-		if cfg.ContextSize == nil {
+		if cfg.ContextSize == nil || *cfg.ContextSize < 0 {
 			ctx := DefaultContextSize
 			cfg.ContextSize = &ctx
 		}
--- a/core/config/hooks_test.go
+++ b/core/config/hooks_test.go
@@ -26,7 +26,7 @@ const (
 // array (tokenizer.ggml.tokens). The big array is exactly what SkipLargeMetadata
 // + UseMMap are expected to avoid reading element-by-element, so it must survive a
 // round-trip through the real hook without corrupting the guessed defaults.
-func writeTestGGUF(path, chatTemplate string, vocab int) error {
+func writeTestGGUF(path, chatTemplate string, vocab int, ctxTrain uint32) error {
 	wStr := func(b *bytes.Buffer, s string) {
 		binary.Write(b, binary.LittleEndian, uint64(len(s)))
 		b.WriteString(s)
@@ -45,7 +45,7 @@ func writeTestGGUF(path, chatTemplate string, vocab int) error {
 	var meta bytes.Buffer
 	kvStr(&meta, "general.architecture", "llama")
 	kvStr(&meta, "general.name", "ReproModel")
-	kvU32(&meta, "llama.context_length", 4096)
+	kvU32(&meta, "llama.context_length", ctxTrain)
 	kvU32(&meta, "llama.attention.head_count", 32)
 	kvU32(&meta, "llama.feed_forward_length", 11008)
 	kvU32(&meta, "llama.block_count", 32)
@@ -211,7 +211,7 @@ var _ = Describe("Backend hooks and parser defaults", func() {
 		It("guesses defaults from a GGUF whose large vocab is skipped", func() {
 			dir := GinkgoT().TempDir()
 			modelFile := "repro.gguf"
-			Expect(writeTestGGUF(filepath.Join(dir, modelFile), chatTemplate, 50000)).To(Succeed())
+			Expect(writeTestGGUF(filepath.Join(dir, modelFile), chatTemplate, 50000, 4096)).To(Succeed())

 			// A pre-set context size short-circuits the GGUF run-estimate, which
 			// needs full tensor info this header-only fixture deliberately omits;
@@ -236,6 +236,26 @@ var _ = Describe("Backend hooks and parser defaults", func() {
 			Expect(cfg.KnownUsecaseStrings).To(ContainElement("FLAG_CHAT"))
 		})

+		It("resolves context_size=-1 to the model's trained maximum context", func() {
+			dir := GinkgoT().TempDir()
+			modelFile := "automax.gguf"
+			// A distinctive trained max proves we read metadata, not the 4096 default.
+			Expect(writeTestGGUF(filepath.Join(dir, modelFile), chatTemplate, 100, 131072)).To(Succeed())
+
+			neg := -1
+			cfg := &ModelConfig{
+				Backend:   "llama-cpp",
+				LLMConfig: LLMConfig{ContextSize: &neg},
+				PredictionOptions: schema.PredictionOptions{
+					BasicModelRequest: schema.BasicModelRequest{Model: modelFile},
+				},
+			}
+			cfg.SetDefaults(ModelPath(dir))
+
+			Expect(cfg.ContextSize).NotTo(BeNil())
+			Expect(*cfg.ContextSize).To(Equal(131072))
+		})
+
 		It("falls back to the default context size when the GGUF is unreadable", func() {
 			dir := GinkgoT().TempDir()
 			Expect(os.WriteFile(filepath.Join(dir, "bad.gguf"), []byte("not a gguf"), 0o644)).To(Succeed())
@@ -254,6 +274,24 @@ var _ = Describe("Backend hooks and parser defaults", func() {
 			Expect(cfg.ContextSize).NotTo(BeNil())
 			Expect(*cfg.ContextSize).To(Equal(DefaultContextSize))
 		})
+
+		It("falls back to the default when context_size=-1 but the GGUF is unreadable", func() {
+			dir := GinkgoT().TempDir()
+			Expect(os.WriteFile(filepath.Join(dir, "bad.gguf"), []byte("not a gguf"), 0o644)).To(Succeed())
+
+			neg := -1
+			cfg := &ModelConfig{
+				Backend:   "llama-cpp",
+				LLMConfig: LLMConfig{ContextSize: &neg},
+				PredictionOptions: schema.PredictionOptions{
+					BasicModelRequest: schema.BasicModelRequest{Model: "bad.gguf"},
+				},
+			}
+			cfg.SetDefaults(ModelPath(dir))
+
+			Expect(cfg.ContextSize).NotTo(BeNil())
+			Expect(*cfg.ContextSize).To(Equal(DefaultContextSize))
+		})
 	})

 	Context("PromptCacheAll default", func() {
--- a/docs/content/advanced/model-configuration.md
+++ b/docs/content/advanced/model-configuration.md
@@ -145,7 +145,7 @@ These settings apply to most LLM backends (llama.cpp, vLLM, etc.):
 | Field | Type | Default | Description |
 |-------|------|---------|-------------|
 | `threads` | int | `processor count` | Number of threads for parallel computation |
-| `context_size` | int | `512` | Maximum context size (number of tokens) |
+| `context_size` | int | `512` | Maximum context size in tokens. Set to `-1` to auto-use the model's full trained context from GGUF metadata (raw max, no VRAM capping; a warning is logged if it may not fit detected VRAM). |
 | `f16` | bool | `false` | Enable 16-bit floating point precision (GPU acceleration) |
 | `gpu_layers` | int | `0` | Number of layers to offload to GPU (0 = CPU only) |

--- a/docs/content/reference/cli-reference.md
+++ b/docs/content/reference/cli-reference.md
@@ -73,7 +73,7 @@ For more information on VRAM management, see [VRAM and Memory Management]({{%rel
 |-----------|---------|-------------|----------------------|
 | `--f16` | `false` | Enable GPU acceleration | `$LOCALAI_F16`, `$F16` |
 | `-t, --threads` | | Number of threads used for parallel computation. Usage of the number of physical cores in the system is suggested | `$LOCALAI_THREADS`, `$THREADS` |
-| `--context-size` | | Default context size for models | `$LOCALAI_CONTEXT_SIZE`, `$CONTEXT_SIZE` |
+| `--context-size` | | Default context size for models (`-1` = each model's full trained context from GGUF metadata) | `$LOCALAI_CONTEXT_SIZE`, `$CONTEXT_SIZE` |

 ## API Flags