fix(studio): use the backend's default window when context_size is unset

A chat on a model whose config has no context_size got a window of zero,
so the harness fell back to its own default for compaction and for the
context meter, while the backend actually runs with its default window.
The meter and the compaction trigger then disagreed with the model.

Report the window that a request can use, the same value that
/v1/models/capabilities gives (backend.EffectiveRequestContextSize). A
model with no local config, such as a remote one, still gives zero.

Assisted-by: Claude:claude-sonnet-5-5 go-test
This commit is contained in:
Ettore Di Giacinto committed 2026-11-03 21:50:57 +00:00
1 parent 06adb4b94e
commit 055c456d9e
2 files changed
+10 -5

No files matched your search

+8 -4
View File
@@ -14,6 +14,7 @@ import (
"github.com/mudler/xlog"
"gorm.io/gorm"
corebackend "github.com/mudler/LocalAI/core/backend"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/http/auth"
mcpendpoint "github.com/mudler/LocalAI/core/http/endpoints/mcp"
@@ -323,14 +324,17 @@ func (a *Application) studioModelConfig(model string) (config.ModelConfig, bool)
return bcl.GetModelConfig(name)
}
// studioContextTokens is the context window of a model, or zero when its config
// does not say.
// studioContextTokens is the context window a request to the model can use. A
// config without context_size runs the backend with its default window, so that
// is the limit (the same one /v1/models/capabilities reports), not a guess of
// the harness. Zero only for a model with no local config (a remote one), whose
// window is not known here.
func (a *Application) studioContextTokens(model string) int {
cfg, ok := a.studioModelConfig(model)
if !ok || cfg.ContextSize == nil {
if !ok {
return 0
}
return *cfg.ContextSize
return corebackend.EffectiveRequestContextSize(cfg)
}
// studioModalities says which inputs a chat model takes natively, from its
+2 -1
View File
@@ -89,7 +89,8 @@ var _ = Describe("studio service at startup", func() {
size := 4096
app.ModelConfigLoader().ReplaceModelConfigs([]config.ModelConfig{{Name: "small", LLMConfig: config.LLMConfig{ContextSize: &size}}, {Name: "unset"}})
Expect(app.studioContextTokens("small")).To(Equal(4096))
Expect(app.studioContextTokens("unset")).To(Equal(0))
// No context_size: the backend runs with its default window.
Expect(app.studioContextTokens("unset")).To(Equal(config.DefaultContextSize))
Expect(app.studioContextTokens("missing")).To(Equal(0))
})