From 055c456d9effb6dd808ab282f2cd2759914fd4a4 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 3 Nov 2026 21:40:53 +0000 Subject: [PATCH] fix(studio): use the backend's default window when context_size is unset A chat on a model whose config has no context_size got a window of zero, so the harness fell back to its own default for compaction and for the context meter, while the backend actually runs with its default window. The meter and the compaction trigger then disagreed with the model. Report the window that a request can use, the same value that /v1/models/capabilities gives (backend.EffectiveRequestContextSize). A model with no local config, such as a remote one, still gives zero. Assisted-by: Claude:claude-sonnet-5-5 go-test --- core/application/studio_service.go | 12 ++++++++---- core/application/studio_service_test.go | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/application/studio_service.go b/core/application/studio_service.go index 6125791912..830dbd9f3c 100644 --- a/core/application/studio_service.go +++ b/core/application/studio_service.go @@ -14,6 +14,7 @@ import ( "github.com/mudler/xlog" "gorm.io/gorm" + corebackend "github.com/mudler/LocalAI/core/backend" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/auth" mcpendpoint "github.com/mudler/LocalAI/core/http/endpoints/mcp" @@ -323,14 +324,17 @@ func (a *Application) studioModelConfig(model string) (config.ModelConfig, bool) return bcl.GetModelConfig(name) } -// studioContextTokens is the context window of a model, or zero when its config -// does not say. +// studioContextTokens is the context window a request to the model can use. A +// config without context_size runs the backend with its default window, so that +// is the limit (the same one /v1/models/capabilities reports), not a guess of +// the harness. Zero only for a model with no local config (a remote one), whose +// window is not known here. func (a *Application) studioContextTokens(model string) int { cfg, ok := a.studioModelConfig(model) - if !ok || cfg.ContextSize == nil { + if !ok { return 0 } - return *cfg.ContextSize + return corebackend.EffectiveRequestContextSize(cfg) } // studioModalities says which inputs a chat model takes natively, from its diff --git a/core/application/studio_service_test.go b/core/application/studio_service_test.go index 8818d02ef7..f56c9c0abd 100644 --- a/core/application/studio_service_test.go +++ b/core/application/studio_service_test.go @@ -89,7 +89,8 @@ var _ = Describe("studio service at startup", func() { size := 4096 app.ModelConfigLoader().ReplaceModelConfigs([]config.ModelConfig{{Name: "small", LLMConfig: config.LLMConfig{ContextSize: &size}}, {Name: "unset"}}) Expect(app.studioContextTokens("small")).To(Equal(4096)) - Expect(app.studioContextTokens("unset")).To(Equal(0)) + // No context_size: the backend runs with its default window. + Expect(app.studioContextTokens("unset")).To(Equal(config.DefaultContextSize)) Expect(app.studioContextTokens("missing")).To(Equal(0)) })