mirror of
https://github.com/mudler/LocalAI.git
synced 2026-11-06 04:23:15 -05:00
fix(studio): use the backend's default window when context_size is unset
A chat on a model whose config has no context_size got a window of zero, so the harness fell back to its own default for compaction and for the context meter, while the backend actually runs with its default window. The meter and the compaction trigger then disagreed with the model. Report the window that a request can use, the same value that /v1/models/capabilities gives (backend.EffectiveRequestContextSize). A model with no local config, such as a remote one, still gives zero. Assisted-by: Claude:claude-sonnet-5-5 go-test
This commit is contained in:
2 files changed
+10
-5
No files matched your search
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/mudler/xlog"
|
||||
"gorm.io/gorm"
|
||||
|
||||
corebackend "github.com/mudler/LocalAI/core/backend"
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/http/auth"
|
||||
mcpendpoint "github.com/mudler/LocalAI/core/http/endpoints/mcp"
|
||||
@@ -323,14 +324,17 @@ func (a *Application) studioModelConfig(model string) (config.ModelConfig, bool)
|
||||
return bcl.GetModelConfig(name)
|
||||
}
|
||||
|
||||
// studioContextTokens is the context window of a model, or zero when its config
|
||||
// does not say.
|
||||
// studioContextTokens is the context window a request to the model can use. A
|
||||
// config without context_size runs the backend with its default window, so that
|
||||
// is the limit (the same one /v1/models/capabilities reports), not a guess of
|
||||
// the harness. Zero only for a model with no local config (a remote one), whose
|
||||
// window is not known here.
|
||||
func (a *Application) studioContextTokens(model string) int {
|
||||
cfg, ok := a.studioModelConfig(model)
|
||||
if !ok || cfg.ContextSize == nil {
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
return *cfg.ContextSize
|
||||
return corebackend.EffectiveRequestContextSize(cfg)
|
||||
}
|
||||
|
||||
// studioModalities says which inputs a chat model takes natively, from its
|
||||
|
||||
@@ -89,7 +89,8 @@ var _ = Describe("studio service at startup", func() {
|
||||
size := 4096
|
||||
app.ModelConfigLoader().ReplaceModelConfigs([]config.ModelConfig{{Name: "small", LLMConfig: config.LLMConfig{ContextSize: &size}}, {Name: "unset"}})
|
||||
Expect(app.studioContextTokens("small")).To(Equal(4096))
|
||||
Expect(app.studioContextTokens("unset")).To(Equal(0))
|
||||
// No context_size: the backend runs with its default window.
|
||||
Expect(app.studioContextTokens("unset")).To(Equal(config.DefaultContextSize))
|
||||
Expect(app.studioContextTokens("missing")).To(Equal(0))
|
||||
})
|
||||
|
||||
|
||||
Reference in new issue
Block a user