mirror of
https://github.com/mudler/LocalAI.git
synced 2026-11-05 20:13:23 -05:00
fix(studio): bound the model call of a compaction
A compaction ran on a context with no deadline, so a model that never answered held the chat busy for good. Give it a 10 minute limit. Assisted-by: Claude Code:claude-sonnet-5-5 Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
1 file changed
+7
-1
@@ -260,6 +260,9 @@ func (s *Service) contextCommand(ctx context.Context, p, conv string, fn func(re
|
||||
return err
|
||||
}
|
||||
|
||||
// compactLimit bounds the model call of a compaction.
|
||||
const compactLimit = 10 * time.Minute
|
||||
|
||||
// Compact shrinks the model's context of the chat now, with the summary the
|
||||
// loop makes when the context fills up, and returns its size in tokens before
|
||||
// and after. The transcript keeps every message and gets a marker; the next
|
||||
@@ -281,7 +284,10 @@ func (s *Service) Compact(ctx context.Context, user, conv string) (before, after
|
||||
s.log.Warn("studio: could not close the chat after a compaction", "chat", conv, "error", cerr)
|
||||
}
|
||||
}()
|
||||
before, after, err = chat.Compact(context.WithoutCancel(ctx))
|
||||
// The summary is one model call that outlives the request; bound it.
|
||||
cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), compactLimit)
|
||||
defer cancel()
|
||||
before, after, err = chat.Compact(cctx)
|
||||
switch {
|
||||
case errors.Is(err, session.ErrNothingToCompact):
|
||||
return ErrNothingToCompact
|
||||
|
||||
Reference in new issue
Block a user