mirror of
https://github.com/ollama/ollama.git
synced 2026-02-07 14:13:29 -05:00
18 lines
326 B
Go
18 lines
326 B
Go
package tokenizer
|
|
|
|
const (
|
|
TOKEN_TYPE_NORMAL = iota + 1
|
|
TOKEN_TYPE_UNKNOWN
|
|
TOKEN_TYPE_CONTROL
|
|
TOKEN_TYPE_USER_DEFINED
|
|
TOKEN_TYPE_UNUSED
|
|
TOKEN_TYPE_BYTE
|
|
)
|
|
|
|
type Tokenizer interface {
|
|
Encode(s string, addSpecial bool) ([]int32, error)
|
|
Decode([]int32) (string, error)
|
|
Is(int32, Special) bool
|
|
Vocabulary() *Vocabulary
|
|
}
|