mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-13 06:45:26 -04:00
localai-org-maint-botandlocalai-org-maint-bot
0761bd02c7
* feat(config): add context compression policy Define the opt-in model configuration contract before the chat middleware consumes it. Document each policy field so later request handling does not invent a second schema.\n\nRefs #9534\n\nAssisted-by: Codex:gpt-5 * fix(config): register compression fields The model editor metadata gate rejects new config fields without descriptions and suitable controls. Register the compression policy so operators can edit its six fields safely. Assisted-by: Codex:gpt-5 [monitoring-prs] * feat(chat): compress long contexts Long conversations currently fail once they reach the model context window. The opt-in policy now summarizes complete older turns before primary inference and preserves the newest tool chains. Both OpenAI and MCP chat routes share the same transformation. Usage metadata and metrics expose each compression event. Refs #9534 Assisted-by: Codex:gpt-5 --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
42 lines
1.4 KiB
Go
42 lines
1.4 KiB
Go
package tokens
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
|
|
"github.com/mudler/LocalAI/core/schema"
|
|
)
|
|
|
|
// CountMessages returns a stable OpenAI-compatible estimate that includes
|
|
// roles, multimodal text, tool calls, and tool results. Exact backend counts
|
|
// are model-specific; the safety margin comes from the configurable trigger.
|
|
func CountMessages(messages []schema.Message) (int, error) {
|
|
total := 0
|
|
for _, message := range messages {
|
|
payload, err := json.Marshal(message)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("encode %s message: %w", message.Role, err)
|
|
}
|
|
// Use a conservative offline estimate. A vocabulary download in the
|
|
// request path can hang firewalled installations, while LocalAI must
|
|
// decide whether to compress before any model is loaded. One token per
|
|
// JSON byte is a safe upper-bound estimate for byte-fallback tokenizers.
|
|
// It intentionally triggers compression early instead of risking a late
|
|
// backend context rejection for code, identifiers, or multilingual text.
|
|
total += 4 + len(payload)
|
|
}
|
|
return total + 2, nil
|
|
}
|
|
|
|
// CountPayload estimates non-message request data such as tool schemas.
|
|
func CountPayload(value any) (int, error) {
|
|
payload, err := json.Marshal(value)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("encode token payload: %w", err)
|
|
}
|
|
if string(payload) == "{}" || string(payload) == "null" {
|
|
return 0, nil
|
|
}
|
|
return len(payload), nil
|
|
}
|