LocalAI/core/services/routing/router/trim.go

package router

import (
	"math"
	"strings"
	"sync"
	"sync/atomic"
	"unicode/utf8"

	"github.com/mudler/xlog"
)

// pretrimRunesPerToken is deliberately high (most text is 3–5 runes/token,
// tokenisers rarely exceed 6) so the cheap rune pre-trim keeps a superset of
// what fits before any tokenize call.
const pretrimRunesPerToken = 6

// tokenBudgetMargin absorbs BPE-boundary drift and the framing tokens a
// renderer adds, so a prompt measured at exactly the budget still fits n_ctx.
const tokenBudgetMargin = 16

// JoinTurns joins per-turn texts oldest→newest with a trailing newline each.
// The probe builder, the trimmer, and every classifier share this so the text
// a model sees has one canonical shape.
func JoinTurns(turns []string) string {
	var b strings.Builder
	for _, m := range turns {
		b.WriteString(m)
		b.WriteByte('\n')
	}
	return b.String()
}

// promptTrimmer fits an oldest→newest turn list into a token budget for one
// model: optimistic rune pre-trim, tokenize once, then recalibrate with the
// real runes/token and drop whole turns oldest-first until the rendered prompt
// fits. The newest turn is never dropped — if it alone overflows it's sent
// whole and the backend's n_ctx guard is the backstop.
//
// render wraps the joined turns into what the model actually tokenizes: a chat
// template for the scorer, identityRender for an embedder/reranker on raw text.
type promptTrimmer struct {
	tokenize func(string) (int, error)
	render   func(joined string) (string, error)
	budget   int
}

func identityRender(s string) (string, error) { return s, nil }

func (t promptTrimmer) fit(turns []string) string {
	if len(turns) == 0 {
		return ""
	}
	kept := turns[runePretrimStart(turns, t.budget*pretrimRunesPerToken):]

	joined := JoinTurns(kept)
	rendered, err := t.render(joined)
	if err != nil {
		return joined
	}
	total, err := t.tokenize(rendered)
	if err != nil || total <= t.budget {
		return joined
	}

	runesPerToken := float64(utf8.RuneCountInString(rendered)) / float64(total)
	if runesPerToken <= 0 {
		runesPerToken = 1
	}
	est := total
	keep := 0
	for keep < len(kept)-1 && est > t.budget {
		est -= int(math.Ceil(float64(utf8.RuneCountInString(kept[keep])) / runesPerToken))
		keep++
	}

	for {
		tail := JoinTurns(kept[keep:])
		rendered, err := t.render(tail)
		if err != nil {
			return tail
		}
		n, err := t.tokenize(rendered)
		if err != nil || n <= t.budget {
			return tail
		}
		if keep >= len(kept)-1 {
			xlog.Warn("router: newest turn alone exceeds model context; sending it whole — backend n_ctx guard is the backstop",
				"tokens", n, "budget", t.budget)
			return tail
		}
		keep++
	}
}

// runePretrimStart returns the oldest index to keep so the joined tail stays
// within budgetRunes. The newest turn is always kept; older ones are added
// while they fit.
func runePretrimStart(turns []string, budgetRunes int) int {
	if budgetRunes <= 0 || len(turns) == 0 {
		return 0
	}
	start := len(turns) - 1
	total := utf8.RuneCountInString(turns[start])
	for i := len(turns) - 2; i >= 0; i-- {
		r := utf8.RuneCountInString(turns[i])
		if total+r > budgetRunes {
			break
		}
		total += r
		start = i
	}
	return start
}

// lazyBudget computes a model's probe token budget once, on first use, caching
// the result: maxContext minus the longest per-call extra (scorer candidates,
// reranker documents; none for a plain embed) minus tokenBudgetMargin. A
// tokenizer error leaves it uncomputed so a transient failure (model still
// loading) recovers on a later call; extras that already fill the context are
// cached as disabled.
type lazyBudget struct {
	tokenize   func(string) (int, error)
	maxContext int
	extras     []string

	mu    sync.Mutex
	value atomic.Int64 // 0=unset, >0=budget, -1=disabled
}

func (l *lazyBudget) get() int {
	if l == nil || l.tokenize == nil || l.maxContext <= 0 {
		return 0
	}
	if v := l.value.Load(); v != 0 {
		if v < 0 {
			return 0
		}
		return int(v)
	}
	l.mu.Lock()
	defer l.mu.Unlock()
	if v := l.value.Load(); v != 0 {
		if v < 0 {
			return 0
		}
		return int(v)
	}
	longest := 0
	for _, e := range l.extras {
		n, err := l.tokenize(e)
		if err != nil {
			return 0 // transient: leave unset so a later call retries
		}
		if n > longest {
			longest = n
		}
	}
	b := l.maxContext - longest - tokenBudgetMargin
	if b <= 0 {
		l.value.Store(-1)
		return 0
	}
	l.value.Store(int64(b))
	return b
}

// trimmedProbeText returns the text to feed a model: the most recent turns
// that fit its token budget, or p.Prompt when trimming is disabled (no
// tokenizer/context wired, or a single-input probe with no Messages).
func trimmedProbeText(p Probe, b *lazyBudget, render func(string) (string, error)) string {
	if len(p.Messages) > 0 {
		if budget := b.get(); budget > 0 {
			return promptTrimmer{tokenize: b.tokenize, render: render, budget: budget}.fit(p.Messages)
		}
	}
	return p.Prompt
}