mirror of
https://github.com/ollama/ollama.git
synced 2026-08-04 12:04:03 -04:00
28fbbb06d5
Generalize the draft path so a head that maintains a KV cache (EAGLE-style) and Gemma's read-only single-position assistant both fit one drafter interface with no per-model branches, and make the committed stream the drafter's maintenance mechanism — every committed run is reported, the drafter pairs each draft slot with its look-ahead token and flushes completed pairs to the draft caches. The draft KV thus stays prefix-cached alongside the target in every session, drafting or not.
47 lines
1.3 KiB
Go
47 lines
1.3 KiB
Go
package batch
|
|
|
|
import "github.com/ollama/ollama/x/mlxrunner/mlx"
|
|
|
|
// Batch is the per-forward-pass input handed to a model.
|
|
type Batch struct {
|
|
// InputIDs is the input token IDs for this forward pass, shape (B, L).
|
|
InputIDs *mlx.Array
|
|
|
|
// SeqOffsets gives each row's current position within its sequence —
|
|
// where the chunk in InputIDs starts. Length equals the batch dimension
|
|
// of InputIDs.
|
|
SeqOffsets []int32
|
|
|
|
// SeqQueryLens is each row's real query length in this forward. Values
|
|
// less than L mean the row's tail is padding that must be masked out.
|
|
// Length equals the batch dimension of InputIDs.
|
|
SeqQueryLens []int32
|
|
|
|
// Hidden is the target hidden state a draft model fuses with its input
|
|
// embedding for this step. It is nil for ordinary forward passes.
|
|
Hidden *mlx.Array
|
|
|
|
// Memo is per-forward memoization used to cache results, such as masks,
|
|
// which are often the same across layers.
|
|
Memo Memo
|
|
}
|
|
|
|
type Memo struct {
|
|
entries map[any]any
|
|
}
|
|
|
|
// Get returns the memoized value for key and true if present, or nil
|
|
// and false otherwise.
|
|
func (m *Memo) Get(key any) (any, bool) {
|
|
v, ok := m.entries[key]
|
|
return v, ok
|
|
}
|
|
|
|
// Put stores value under key, allocating on first use.
|
|
func (m *Memo) Put(key, value any) {
|
|
if m.entries == nil {
|
|
m.entries = map[any]any{}
|
|
}
|
|
m.entries[key] = value
|
|
}
|