diff --git a/core/application/studio_service.go b/core/application/studio_service.go index 6125791912..830dbd9f3c 100644 --- a/core/application/studio_service.go +++ b/core/application/studio_service.go @@ -14,6 +14,7 @@ import ( "github.com/mudler/xlog" "gorm.io/gorm" + corebackend "github.com/mudler/LocalAI/core/backend" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/auth" mcpendpoint "github.com/mudler/LocalAI/core/http/endpoints/mcp" @@ -323,14 +324,17 @@ func (a *Application) studioModelConfig(model string) (config.ModelConfig, bool) return bcl.GetModelConfig(name) } -// studioContextTokens is the context window of a model, or zero when its config -// does not say. +// studioContextTokens is the context window a request to the model can use. A +// config without context_size runs the backend with its default window, so that +// is the limit (the same one /v1/models/capabilities reports), not a guess of +// the harness. Zero only for a model with no local config (a remote one), whose +// window is not known here. func (a *Application) studioContextTokens(model string) int { cfg, ok := a.studioModelConfig(model) - if !ok || cfg.ContextSize == nil { + if !ok { return 0 } - return *cfg.ContextSize + return corebackend.EffectiveRequestContextSize(cfg) } // studioModalities says which inputs a chat model takes natively, from its diff --git a/core/application/studio_service_test.go b/core/application/studio_service_test.go index 8818d02ef7..f56c9c0abd 100644 --- a/core/application/studio_service_test.go +++ b/core/application/studio_service_test.go @@ -89,7 +89,8 @@ var _ = Describe("studio service at startup", func() { size := 4096 app.ModelConfigLoader().ReplaceModelConfigs([]config.ModelConfig{{Name: "small", LLMConfig: config.LLMConfig{ContextSize: &size}}, {Name: "unset"}}) Expect(app.studioContextTokens("small")).To(Equal(4096)) - Expect(app.studioContextTokens("unset")).To(Equal(0)) + // No context_size: the backend runs with its default window. + Expect(app.studioContextTokens("unset")).To(Equal(config.DefaultContextSize)) Expect(app.studioContextTokens("missing")).To(Equal(0)) })