server: allow running embed models in parallel

The ability to run embedding models in parallel with other types of models
was removed due to limitations in server slot loading in a past version of
the server. This slot loading system is no longer used, and embedding models
can run in parallel with chat models.

README.md

@@ -54,10 +54,6 @@ Here are some example models that can be downloaded:
 
 | Model              | Parameters | Size  | Download                         |
 | ------------------ | ---------- | ----- | -------------------------------- |
-| Gemma 3            | 1B         | 815MB | `ollama run gemma3:1b`           |
-| Gemma 3            | 4B         | 3.3GB | `ollama run gemma3`              |
-| Gemma 3            | 12B        | 8.1GB | `ollama run gemma3:12b`          |
-| Gemma 3            | 27B        | 17GB  | `ollama run gemma3:27b`          |
 | QwQ                | 32B        | 20GB  | `ollama run qwq`                 |
 | DeepSeek-R1        | 7B         | 4.7GB | `ollama run deepseek-r1`         |
 | DeepSeek-R1        | 671B       | 404GB | `ollama run deepseek-r1:671b`    |
@@ -70,6 +66,9 @@ Here are some example models that can be downloaded:
 | Llama 3.1          | 405B       | 231GB | `ollama run llama3.1:405b`       |
 | Phi 4              | 14B        | 9.1GB | `ollama run phi4`                |
 | Phi 4 Mini         | 3.8B       | 2.5GB | `ollama run phi4-mini`           |
+| Gemma 2            | 2B         | 1.6GB | `ollama run gemma2:2b`           |
+| Gemma 2            | 9B         | 5.5GB | `ollama run gemma2`              |
+| Gemma 2            | 27B        | 16GB  | `ollama run gemma2:27b`          |
 | Mistral            | 7B         | 4.1GB | `ollama run mistral`             |
 | Moondream 2        | 1.4B       | 829MB | `ollama run moondream`           |
 | Neural Chat        | 7B         | 4.1GB | `ollama run neural-chat`         |
@@ -572,7 +571,6 @@ See the [API documentation](./docs/api.md) for all endpoints.
 - [llama.cpp](https://github.com/ggerganov/llama.cpp) project founded by Georgi Gerganov.
 
 ### Observability
-- [Opik](https://www.comet.com/docs/opik/cookbook/ollama) is an open-source platform to debug, evaluate, and monitor your LLM applications, RAG systems, and agentic workflows with comprehensive tracing, automated evaluations, and production-ready dashboards. Opik supports native intergration to Ollama.
 - [Lunary](https://lunary.ai/docs/integrations/ollama) is the leading open-source LLM observability platform. It provides a variety of enterprise-grade features such as real-time analytics, prompt templates management, PII masking, and comprehensive agent tracing.
 - [OpenLIT](https://github.com/openlit/openlit) is an OpenTelemetry-native tool for monitoring Ollama Applications & GPUs using traces and metrics.
 - [HoneyHive](https://docs.honeyhive.ai/integrations/ollama) is an AI observability and evaluation platform for AI agents. Use HoneyHive to evaluate agent performance, interrogate failures, and monitor quality in production.

api/types.go

@@ -349,7 +349,6 @@ type ShowResponse struct {
 	Messages      []Message      `json:"messages,omitempty"`
 	ModelInfo     map[string]any `json:"model_info,omitempty"`
 	ProjectorInfo map[string]any `json:"projector_info,omitempty"`
-	Tensors       []Tensor       `json:"tensors,omitempty"`
 	ModifiedAt    time.Time      `json:"modified_at,omitempty"`
 }
 
@@ -468,13 +467,6 @@ type ModelDetails struct {
 	QuantizationLevel string   `json:"quantization_level"`
 }
 
-// Tensor describes the metadata for a given tensor.
-type Tensor struct {
-	Name  string   `json:"name"`
-	Type  string   `json:"type"`
-	Shape []uint64 `json:"shape"`
-}
-
 func (m *Metrics) Summary() {
 	if m.TotalDuration > 0 {
 		fmt.Fprintf(os.Stderr, "total duration:       %v\n", m.TotalDuration)

cmd/cmd.go

@@ -18,7 +18,6 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -569,9 +568,8 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
 	parameters, errParams := cmd.Flags().GetBool("parameters")
 	system, errSystem := cmd.Flags().GetBool("system")
 	template, errTemplate := cmd.Flags().GetBool("template")
-	verbose, errVerbose := cmd.Flags().GetBool("verbose")
 
-	for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate, errVerbose} {
+	for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate} {
 		if boolErr != nil {
 			return errors.New("error retrieving flags")
 		}
@@ -609,7 +607,7 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
 		return errors.New("only one of '--license', '--modelfile', '--parameters', '--system', or '--template' can be specified")
 	}
 
-	req := api.ShowRequest{Name: args[0], Verbose: verbose}
+	req := api.ShowRequest{Name: args[0]}
 	resp, err := client.Show(cmd.Context(), &req)
 	if err != nil {
 		return err
@@ -632,10 +630,10 @@ func ShowHandler(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	return showInfo(resp, verbose, os.Stdout)
+	return showInfo(resp, os.Stdout)
 }
 
-func showInfo(resp *api.ShowResponse, verbose bool, w io.Writer) error {
+func showInfo(resp *api.ShowResponse, w io.Writer) error {
 	tableRender := func(header string, rows func() [][]string) {
 		fmt.Fprintln(w, " ", header)
 		table := tablewriter.NewWriter(w)
@@ -692,45 +690,6 @@ func showInfo(resp *api.ShowResponse, verbose bool, w io.Writer) error {
 		})
 	}
 
-	if resp.ModelInfo != nil && verbose {
-		tableRender("Metadata", func() (rows [][]string) {
-			keys := make([]string, 0, len(resp.ModelInfo))
-			for k := range resp.ModelInfo {
-				keys = append(keys, k)
-			}
-			sort.Strings(keys)
-
-			for _, k := range keys {
-				var v string
-				switch vData := resp.ModelInfo[k].(type) {
-				case string:
-					v = vData
-				case float64:
-					v = fmt.Sprintf("%g", vData)
-				case []any:
-					n := 3
-					if len(vData) < n {
-						n = len(vData)
-					}
-					v = fmt.Sprintf("%v", vData[:n])
-				default:
-					v = fmt.Sprintf("%T", vData)
-				}
-				rows = append(rows, []string{"", k, v})
-			}
-			return
-		})
-	}
-
-	if len(resp.Tensors) > 0 && verbose {
-		tableRender("Tensors", func() (rows [][]string) {
-			for _, t := range resp.Tensors {
-				rows = append(rows, []string{"", t.Name, t.Type, fmt.Sprint(t.Shape)})
-			}
-			return
-		})
-	}
-
 	head := func(s string, n int) (rows [][]string) {
 		scanner := bufio.NewScanner(strings.NewReader(s))
 		for scanner.Scan() && (len(rows) < n || n < 0) {
@@ -1237,7 +1196,6 @@ func NewCLI() *cobra.Command {
 	showCmd.Flags().Bool("parameters", false, "Show parameters of a model")
 	showCmd.Flags().Bool("template", false, "Show template of a model")
 	showCmd.Flags().Bool("system", false, "Show system message of a model")
-	showCmd.Flags().BoolP("verbose", "v", false, "Show detailed model information")
 
 	runCmd := &cobra.Command{
 		Use:     "run MODEL [PROMPT]",

cmd/cmd_test.go

@@ -27,7 +27,7 @@ func TestShowInfo(t *testing.T) {
 				ParameterSize:     "7B",
 				QuantizationLevel: "FP16",
 			},
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -57,7 +57,7 @@ func TestShowInfo(t *testing.T) {
 				ParameterSize:     "7B",
 				QuantizationLevel: "FP16",
 			},
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -68,56 +68,6 @@ func TestShowInfo(t *testing.T) {
     embedding length    0       
     quantization        FP16    
 
-`
-		if diff := cmp.Diff(expect, b.String()); diff != "" {
-			t.Errorf("unexpected output (-want +got):\n%s", diff)
-		}
-	})
-
-	t.Run("verbose model", func(t *testing.T) {
-		var b bytes.Buffer
-		if err := showInfo(&api.ShowResponse{
-			Details: api.ModelDetails{
-				Family:            "test",
-				ParameterSize:     "8B",
-				QuantizationLevel: "FP16",
-			},
-			Parameters: `
-			stop up`,
-			ModelInfo: map[string]any{
-				"general.architecture":    "test",
-				"general.parameter_count": float64(8_000_000_000),
-				"test.context_length":     float64(1000),
-				"test.embedding_length":   float64(11434),
-			},
-			Tensors: []api.Tensor{
-				{Name: "blk.0.attn_k.weight", Type: "BF16", Shape: []uint64{42, 3117}},
-				{Name: "blk.0.attn_q.weight", Type: "FP16", Shape: []uint64{3117, 42}},
-			},
-		}, true, &b); err != nil {
-			t.Fatal(err)
-		}
-
-		expect := `  Model
-    architecture        test     
-    parameters          8B       
-    context length      1000     
-    embedding length    11434    
-    quantization        FP16     
-
-  Parameters
-    stop    up    
-
-  Metadata
-    general.architecture       test     
-    general.parameter_count    8e+09    
-    test.context_length        1000     
-    test.embedding_length      11434    
-
-  Tensors
-    blk.0.attn_k.weight    BF16    [42 3117]    
-    blk.0.attn_q.weight    FP16    [3117 42]    
-
 `
 		if diff := cmp.Diff(expect, b.String()); diff != "" {
 			t.Errorf("unexpected output (-want +got):\n%s", diff)
@@ -139,7 +89,7 @@ func TestShowInfo(t *testing.T) {
 			stop you
 			stop up
 			temperature 99`,
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -176,7 +126,7 @@ func TestShowInfo(t *testing.T) {
 				"clip.vision.embedding_length": float64(0),
 				"clip.vision.projection_dim":   float64(0),
 			},
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -209,7 +159,7 @@ func TestShowInfo(t *testing.T) {
 Ahoy, matey!
 Weigh anchor!
 			`,
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 
@@ -238,7 +188,7 @@ Weigh anchor!
 				QuantizationLevel: "FP16",
 			},
 			License: license,
-		}, false, &b); err != nil {
+		}, &b); err != nil {
 			t.Fatal(err)
 		}
 

cmd/interactive.go

@@ -195,10 +195,6 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 			opts.Messages = []api.Message{}
 			fmt.Printf("Loading model '%s'\n", opts.Model)
 			if err := loadOrUnloadModel(cmd, &opts); err != nil {
-				if strings.Contains(err.Error(), "not found") {
-					fmt.Printf("error: %v\n", err)
-					continue
-				}
 				return err
 			}
 			continue
@@ -347,7 +343,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 
 				switch args[1] {
 				case "info":
-					_ = showInfo(resp, false, os.Stderr)
+					_ = showInfo(resp, os.Stderr)
 				case "license":
 					if resp.License == "" {
 						fmt.Println("No license was specified for this model.")

convert/convert.go

@@ -13,13 +13,8 @@ import (
 )
 
 type ModelParameters struct {
-	Architectures []string       `json:"architectures"`
-	VocabSize     uint32         `json:"vocab_size"`
-	TextModel     TextParameters `json:"text_config"`
-}
-
-type TextParameters struct {
-	VocabSize uint32 `json:"vocab_size"`
+	Architectures []string `json:"architectures"`
+	VocabSize     uint32   `json:"vocab_size"`
 }
 
 type AdapterParameters struct {
@@ -190,8 +185,6 @@ func ConvertModel(fsys fs.FS, ws io.WriteSeeker) error {
 		conv = &gemmaModel{}
 	case "Gemma2ForCausalLM":
 		conv = &gemma2Model{}
-	case "Gemma3ForCausalLM", "Gemma3ForConditionalGeneration":
-		conv = &gemma3Model{Architecture: p.Architectures[0]}
 	case "Phi3ForCausalLM":
 		conv = &phi3Model{}
 	case "Qwen2ForCausalLM":
@@ -220,14 +213,7 @@ func ConvertModel(fsys fs.FS, ws io.WriteSeeker) error {
 	}
 
 	vocabSize := int(p.VocabSize)
-	if vocabSize == 0 {
-		tVocabSize := int(p.TextModel.VocabSize)
-		vocabSize = tVocabSize
-	}
-
 	switch {
-	case vocabSize == 0:
-		slog.Warn("vocabulary size was not explicitly set by the model", "default size", len(t.Vocabulary.Tokens))
 	case vocabSize > len(t.Vocabulary.Tokens):
 		slog.Warn("vocabulary is smaller than expected, padding with dummy tokens", "expect", vocabSize, "actual", len(t.Vocabulary.Tokens))
 		for i := range vocabSize - len(t.Vocabulary.Tokens) {

convert/convert_gemma.go

@@ -45,7 +45,7 @@ func (p *gemmaModel) KV(t *Tokenizer) ggml.KV {
 func (p *gemmaModel) Tensors(ts []Tensor) []ggml.Tensor {
 	var out []ggml.Tensor
 	for _, t := range ts {
-		if !strings.HasPrefix(t.Name(), "v.") && strings.HasSuffix(t.Name(), "_norm.weight") {
+		if strings.HasSuffix(t.Name(), "_norm.weight") {
 			t.SetRepacker(p.addOne)
 		}
 

convert/convert_gemma3.go

@@ -1,142 +0,0 @@
-package convert
-
-import (
-	"cmp"
-
-	"github.com/ollama/ollama/fs/ggml"
-)
-
-type gemma3Model struct {
-	gemmaModel
-	Architecture string
-	TextModel    struct {
-		HeadDim          uint32 `json:"head_dim"`
-		HiddenSize       uint32 `json:"hidden_size"`
-		HiddenLayers     uint32 `json:"num_hidden_layers"`
-		IntermediateSize uint32 `json:"intermediate_size"`
-		SlidingWindow    uint32 `json:"sliding_window"`
-	} `json:"text_config"`
-	VisionModel struct {
-		NumAttentionHeads uint32  `json:"num_attention_heads"` // attention.head_count 16
-		LayerNormEpsilon  float32 `json:"layer_norm_eps"`      // attention.layer_norm_epsilon 1e-05
-		NumHiddenLayers   uint32  `json:"num_hidden_layers"`   // block_count 32
-		HiddenSize        uint32  `json:"hidden_size"`         // embedding_length 1280
-		IntermediateSize  uint32  `json:"intermediate_size"`   // feed_forward_length 5120
-		ImageSize         uint32  `json:"image_size"`          // image_size 560
-		NumChannels       uint32  `json:"num_channels"`        // num_channels 3
-		PatchSize         uint32  `json:"patch_size"`          // patch_size 14
-	} `json:"vision_config"`
-	MaxPositionEmbeddings    uint32  `json:"max_position_embeddings"`
-	NumAttentionHeads        uint32  `json:"num_attention_heads"`
-	NumKeyValueHeads         uint32  `json:"num_key_value_heads"`
-	RMSNormEPS               float32 `json:"rms_norm_eps"`
-	HeadDim                  uint32  `json:"head_dim"`
-	FinalLogitSoftcap        float32 `json:"final_logit_softcapping"`
-	RopeLocalTheta           float32 `json:"rope_local_base_freq"`
-	RopeGlobalTheta          float32 `json:"rope_global_base_freq"`
-	SlidingWindow            uint32  `json:"sliding_window"`
-	MultiModalTokensPerImage uint32  `json:"mm_tokens_per_image"`
-}
-
-const (
-	gemma4BLayerCount  = 34
-	gemma12BLayerCount = 48
-	gemma27BLayerCount = 62
-)
-
-func (p *gemma3Model) KV(t *Tokenizer) ggml.KV {
-	kv := p.ModelParameters.KV(t)
-	kv["general.architecture"] = "gemma3"
-
-	numBlocks := cmp.Or(p.HiddenLayers, p.TextModel.HiddenLayers)
-	kv["gemma3.block_count"] = numBlocks
-
-	var (
-		numHeads   uint32
-		numKVHeads uint32
-	)
-
-	switch numBlocks {
-	case gemma4BLayerCount:
-		numHeads = 8
-		numKVHeads = 4
-	case gemma12BLayerCount:
-		numHeads = 16
-		numKVHeads = 8
-	case gemma27BLayerCount:
-		numHeads = 32
-		numKVHeads = 16
-	default:
-		numHeads = p.NumAttentionHeads
-		numKVHeads = p.NumKeyValueHeads
-	}
-
-	kv["gemma3.attention.head_count"] = numHeads
-	kv["gemma3.attention.head_count_kv"] = numKVHeads
-
-	switch p.Architecture {
-	case "Gemma3ForCausalLM":
-		kv["gemma3.context_length"] = p.MaxPositionEmbeddings
-		kv["gemma3.attention.layer_norm_rms_epsilon"] = p.RMSNormEPS
-		kv["gemma3.attention.key_length"] = p.HeadDim
-		kv["gemma3.attention.value_length"] = p.HeadDim
-		kv["gemma3.attention.sliding_window"] = p.SlidingWindow
-		kv["gemma3.final_logit_softcapping"] = cmp.Or(p.FinalLogitSoftcap, 30)
-		kv["gemma3.rope.local.freq_base"] = cmp.Or(p.RopeLocalTheta, 10000.0)
-		kv["gemma3.rope.global.freq_base"] = cmp.Or(p.RopeGlobalTheta, 1000000.0)
-		kv["gemma3.embedding_length"] = p.HiddenSize
-		kv["gemma3.feed_forward_length"] = p.IntermediateSize
-	default:
-		kv["gemma3.context_length"] = cmp.Or(p.MaxPositionEmbeddings, 131072)
-		kv["gemma3.embedding_length"] = p.TextModel.HiddenSize
-		kv["gemma3.feed_forward_length"] = p.TextModel.IntermediateSize
-		kv["gemma3.attention.sliding_window"] = p.TextModel.SlidingWindow
-		kv["gemma3.vision.block_count"] = p.VisionModel.NumHiddenLayers
-		kv["gemma3.vision.embedding_length"] = p.VisionModel.HiddenSize
-		kv["gemma3.vision.feed_forward_length"] = p.VisionModel.IntermediateSize
-		kv["gemma3.vision.image_size"] = p.VisionModel.ImageSize
-		kv["gemma3.vision.patch_size"] = p.VisionModel.PatchSize
-		kv["gemma3.vision.num_channels"] = cmp.Or(p.VisionModel.NumChannels, 3)
-		kv["gemma3.vision.attention.head_count"] = p.VisionModel.NumAttentionHeads
-		kv["gemma3.vision.attention.layer_norm_epsilon"] = cmp.Or(p.VisionModel.LayerNormEpsilon, 1e-6)
-		kv["gemma3.attention.key_length"] = cmp.Or(p.TextModel.HeadDim, 256)
-		kv["gemma3.attention.value_length"] = cmp.Or(p.TextModel.HeadDim, 256)
-	}
-
-	if p.MultiModalTokensPerImage > 0 {
-		kv["gemma3.mm.tokens_per_image"] = p.MultiModalTokensPerImage
-	}
-
-	return kv
-}
-
-func (p *gemma3Model) Replacements() []string {
-	return []string{
-		"lm_head", "output",
-		"model.embed_tokens", "token_embd",
-		"model.norm", "output_norm",
-		"vision_tower.vision_model.embeddings", "v",
-		"vision_tower.vision_model", "v",
-		"vision_model.vision_model.embeddings", "v",
-		"vision_model.vision_model", "v",
-		"language_model.", "",
-		"model.layers", "blk",
-		"encoder.layers", "blk",
-		"input_layernorm", "attn_norm",
-		"self_attn.q_proj", "attn_q",
-		"self_attn.q_norm", "attn_q_norm",
-		"self_attn.k_proj", "attn_k",
-		"self_attn.k_norm", "attn_k_norm",
-		"self_attn.v_proj", "attn_v",
-		"self_attn.o_proj", "attn_output",
-		"self_attn.out_proj", "attn_output",
-		"mlp.gate_proj", "ffn_gate",
-		"mlp.down_proj", "ffn_down",
-		"mlp.up_proj", "ffn_up",
-		"post_attention_layernorm", "post_attention_norm",
-		"pre_feedforward_layernorm", "ffn_norm",
-		"post_feedforward_layernorm", "post_ffw_norm",
-		"input_projection_weight", "input_projection.weight",
-		"multi_modal_projector", "mm",
-	}
-}

convert/tokenizer_spm.go

@@ -6,9 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
-	"log/slog"
 	"os"
-	"reflect"
 	"slices"
 
 	"google.golang.org/protobuf/proto"
@@ -17,8 +15,6 @@ import (
 )
 
 func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {
-	slog.Debug("using spm vocabulary")
-
 	ast, err := parseAdditionalSpecialTokens(fsys)
 	if err != nil {
 		return nil, err
@@ -47,19 +43,10 @@ func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {
 			v.Types = append(v.Types, int32(t))
 		default:
 			tt := int32(sentencepiece.ModelProto_SentencePiece_NORMAL)
-
-			// temporary fix to handle gemma3 broken configs
-			if slices.Contains([]string{"<end_of_turn>", "<start_of_turn>"}, piece.GetPiece()) {
+			if slices.Contains(ast, piece.GetPiece()) {
 				tt = int32(sentencepiece.ModelProto_SentencePiece_CONTROL)
 			}
 
-			for _, t := range ast {
-				if t.Content == piece.GetPiece() {
-					tt = int32(sentencepiece.ModelProto_SentencePiece_CONTROL)
-					break
-				}
-			}
-
 			v.Types = append(v.Types, tt)
 		}
 	}
@@ -91,16 +78,10 @@ func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {
 		return cmp.Compare(i.id, j.id)
 	})
 
-	for _, t := range ts {
-		if t.id < len(v.Tokens) {
-			if v.Tokens[t.id] == t.content {
-				slog.Warn("tokenizer", "duplicate token", t.content, "id", t.id)
-				continue
-			}
-			return nil, fmt.Errorf("token mismatch: %s != %s at pos [%d]", t.content, v.Tokens[t.id], t.id)
-		}
-		if t.id != len(v.Tokens) {
-			return nil, fmt.Errorf("invalid token id: [%d] as pos [%d]", t.id, len(v.Tokens))
+	n := len(v.Tokens)
+	for i, t := range ts {
+		if t.id != i+n {
+			return nil, fmt.Errorf("invalid token id: %d", t.id)
 		}
 
 		v.Tokens = append(v.Tokens, t.content)
@@ -111,15 +92,7 @@ func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {
 	return &v, nil
 }
 
-type specialToken struct {
-	Content    string `json:"content"`
-	Lstrip     bool   `json:"lstrip"`
-	Normalized bool   `json:"normalized"`
-	Rstrip     bool   `json:"rstrip"`
-	SingleWord bool   `json:"single_word"`
-}
-
-func parseAdditionalSpecialTokens(fsys fs.FS) ([]specialToken, error) {
+func parseAdditionalSpecialTokens(fsys fs.FS) ([]string, error) {
 	f, err := fsys.Open("special_tokens_map.json")
 	if errors.Is(err, os.ErrNotExist) {
 		return nil, nil
@@ -129,43 +102,12 @@ func parseAdditionalSpecialTokens(fsys fs.FS) ([]specialToken, error) {
 	defer f.Close()
 
 	var m struct {
-		AdditionalSpecialTokens any `json:"additional_special_tokens"`
+		AdditionalSpecialTokens []string `json:"additional_special_tokens"`
 	}
 
 	if err := json.NewDecoder(f).Decode(&m); err != nil {
 		return nil, err
 	}
 
-	var ast []specialToken
-
-	switch st := m.AdditionalSpecialTokens.(type) {
-	case []string:
-		for _, s := range st {
-			ast = append(ast, specialToken{Content: s})
-		}
-	case []any:
-		for _, s := range st {
-			// marshal and unmarshal the object to get the special token
-			tMap := s.(map[string]any)
-			data, err := json.Marshal(tMap)
-			if err != nil {
-				return nil, err
-			}
-
-			var token specialToken
-			err = json.Unmarshal(data, &token)
-			if err != nil {
-				return nil, err
-			}
-
-			ast = append(ast, token)
-		}
-
-	default:
-		slog.Warn("special token", "unknown token", reflect.TypeOf(st))
-	}
-
-	slog.Debug("spm tokenizer", "additional tokens", ast)
-
-	return ast, nil
+	return m.AdditionalSpecialTokens, nil
 }

docs/faq.md

@@ -187,13 +187,6 @@ cloudflared tunnel --url http://localhost:11434 --http-host-header="localhost:11
 
 Ollama allows cross-origin requests from `127.0.0.1` and `0.0.0.0` by default. Additional origins can be configured with `OLLAMA_ORIGINS`.
 
-For browser extensions, you'll need to explicitly allow the extension's origin pattern. Set `OLLAMA_ORIGINS` to include `chrome-extension://*`, `moz-extension://*`, and `safari-web-extension://*` if you wish to allow all browser extensions access, or specific extensions as needed:
-
-```
-# Allow all Chrome, Firefox, and Safari extensions
-OLLAMA_ORIGINS=chrome-extension://*,moz-extension://*,safari-web-extension://* ollama serve
-```
-
 Refer to the section [above](#how-do-i-configure-ollama-server) for how to set environment variables on your platform.
 
 ## Where are models stored?

docs/linux.md

@@ -75,7 +75,7 @@ RestartSec=3
 Environment="PATH=$PATH"
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=default.target
 ```
 
 Then start the service:

fs/ggml/ggml.go

@@ -124,19 +124,6 @@ func (kv KV) Uints(key string, defaultValue ...[]uint32) []uint32 {
 	return s
 }
 
-func (kv KV) Floats(key string, defaultValue ...[]float32) []float32 {
-	r := keyValue(kv, key, &array{})
-	s := make([]float32, r.size)
-	for i := range r.size {
-		s[i] = float32(r.values[i].(float32))
-	}
-	return s
-}
-
-func (kv KV) OllamaEngineRequired() bool {
-	return kv.Architecture() == "gemma3"
-}
-
 func keyValue[T string | uint32 | uint64 | float32 | *array | bool](kv KV, key string, defaultValue ...T) T {
 	if !strings.HasPrefix(key, "tokenizer.") && !strings.HasPrefix(key, "general.") {
 		key = kv.Architecture() + "." + key
@@ -327,10 +314,6 @@ func (t Tensor) Size() uint64 {
 	return t.parameters() * t.typeSize() / t.blockSize()
 }
 
-func (t Tensor) Type() string {
-	return fileType(t.Kind).String()
-}
-
 type container interface {
 	Name() string
 	Decode(io.ReadSeeker) (model, error)
@@ -493,7 +476,7 @@ func (f GGML) GraphSize(context, batch uint64, kvCacheType string) (kv, partialO
 			// vocab graph
 			4*batch*(embedding+vocab)+embedding*vocab*105/128,
 		)
-	case "gemma", "gemma2", "gemma3":
+	case "gemma", "gemma2":
 		fullOffload = max(
 			4*batch*(embedding+vocab),
 			4*batch*(2+context+context*heads+2*embedding+2*embeddingHeadsK*heads),
@@ -583,52 +566,39 @@ func (f GGML) GraphSize(context, batch uint64, kvCacheType string) (kv, partialO
 }
 
 func (llm GGML) VisionGraphSize() (weights, graphSize uint64) {
-	if llm.KV().Uint("vision.block_count") == 0 {
-		return
-	}
-
-	for name, layer := range llm.Tensors().GroupLayers() {
-		if name == "v" || strings.HasPrefix(name, "v.") {
-			for _, tensor := range layer {
-				weights += tensor.Size()
-			}
-		}
-	}
-
-	imageSize := uint64(llm.KV().Uint("vision.image_size"))
-	patchSize := uint64(llm.KV().Uint("vision.patch_size"))
-	if patchSize == 0 {
-		slog.Warn("unknown patch size for vision model")
-		return
-	}
-
-	numChannels := uint64(llm.KV().Uint("vision.num_channels"))
-
-	numPatches := (imageSize / patchSize) * (imageSize / patchSize)
-	if _, ok := llm.Tensors().GroupLayers()["v"]["class_embd"]; ok {
-		numPatches++
-	}
-
-	headCount := uint64(llm.KV().Uint("vision.attention.head_count"))
-	embeddingLength := uint64(llm.KV().Uint("vision.embedding_length"))
-
 	switch llm.KV().Architecture() {
 	case "mllama":
+		for _, layer := range llm.Tensors().GroupLayers()["v"] {
+			weights += layer.Size()
+		}
+
+		kv := func(n string) uint64 {
+			if v, ok := llm.KV()["mllama.vision."+n].(uint32); ok {
+				return uint64(v)
+			}
+
+			return 0
+		}
+
+		imageSize := kv("image_size")
+
+		maxNumTiles := kv("max_num_tiles")
+		embeddingLength := kv("embedding_length")
+		headCount := kv("attention.head_count")
+
+		numPatches := (imageSize / kv("patch_size")) * (imageSize / kv("patch_size"))
+		if _, ok := llm.Tensors().GroupLayers()["v"]["class_embd"]; ok {
+			numPatches++
+		}
+
 		numPaddedPatches := numPatches + 8 - (numPatches%8)%8
 
-		maxNumTiles := uint64(llm.KV().Uint("vision.max_num_tiles"))
-
 		graphSize = 4 * (8 +
-			imageSize*imageSize*numChannels*maxNumTiles +
+			imageSize*imageSize*kv("num_channels")*maxNumTiles +
 			embeddingLength*numPatches*maxNumTiles +
 			9*embeddingLength*numPaddedPatches*maxNumTiles +
 			numPaddedPatches*maxNumTiles*numPaddedPatches*maxNumTiles*headCount)
-	case "gemma3":
-		graphSize = 4 * (imageSize*imageSize*numChannels +
-			embeddingLength*patchSize +
-			numPatches*numPatches*headCount)
 	}
-
 	return weights, graphSize
 }
 

kvcache/causal.go

@@ -21,10 +21,9 @@ type shiftFn func(ctx ml.Context, layer int, key, shift ml.Tensor) (ml.Tensor, e
 type Causal struct {
 	DType      ml.DType
 	Capacity   int32
+	causal     bool
 	windowSize int32
 
-	opts CausalOptions
-
 	// config controls mostly backend-specific optimizations
 	config *ml.CacheConfig
 
@@ -80,6 +79,7 @@ type cellRange struct {
 
 func NewCausalCache(shift shiftFn) *Causal {
 	return &Causal{
+		causal:     true,
 		windowSize: math.MaxInt32,
 		shiftFn:    shift,
 		ctxs:       make(map[int]ml.Context),
@@ -90,6 +90,7 @@ func NewCausalCache(shift shiftFn) *Causal {
 
 func NewSWACache(windowSize int32, shift shiftFn) *Causal {
 	return &Causal{
+		causal:     true,
 		windowSize: windowSize,
 		shiftFn:    shift,
 		ctxs:       make(map[int]ml.Context),
@@ -144,7 +145,6 @@ func (c *Causal) StartForward(ctx ml.Context, opts input.Options) error {
 	c.curBatchSize = len(opts.Positions)
 	c.curSequences = opts.Sequences
 	c.curPositions = opts.Positions
-	c.opts.Except = nil
 
 	var err error
 	c.curLoc, err = c.findStartLoc()
@@ -235,10 +235,9 @@ func (c *Causal) buildMask(ctx ml.Context) (ml.Tensor, error) {
 	mask := make([]float32, batchSize*length)
 
 	for i := range c.curBatchSize {
-		enabled := !slices.Contains(c.opts.Except, i)
 		for j := c.curCellRange.min; j <= c.curCellRange.max; j++ {
 			if !slices.Contains(c.cells[j].sequences, c.curSequences[i]) ||
-				(enabled && c.cells[j].pos > c.curPositions[i]) ||
+				(c.causal && c.cells[j].pos > c.curPositions[i]) ||
 				c.cells[j].pos < c.curPositions[i]-c.windowSize {
 				mask[i*length+(j-c.curCellRange.min)] = float32(math.Inf(-1))
 			}
@@ -405,16 +404,15 @@ func (c *Causal) SetLayer(layer int) {
 	c.curLayer = layer
 }
 
-type CausalOptions struct {
-	// Enabled controls whether the causal mask is generated for a particular index in a batch
-	Except []int
-}
+// SetCausal enables or disables causal mask generation for subsequent calls to Get.
+// This state carries over to future forward passes. The default value is true.
+//
+// ctx may be set to nil if this is called from outside of a forward pass, for
+// example, when initializing the cache.
+func (c *Causal) SetCausal(ctx ml.Context, causal bool) {
+	if c.causal != causal {
+		c.causal = causal
 
-// SetCausal disables causal mask generation for a particular range of indicies in
-// the current batch for subsequent calls to Get. The state resets for the next forward pass.
-func (c *Causal) SetCausal(ctx ml.Context, opts CausalOptions) {
-	if !slices.Equal(c.opts.Except, opts.Except) {
-		c.opts = opts
 		if ctx != nil {
 			var err error
 			c.curMask, err = c.buildMask(ctx)

kvcache/causal_test.go

@@ -441,19 +441,11 @@ func (t *testTensor) Scale(ctx ml.Context, s float64) ml.Tensor {
 	panic("not implemented")
 }
 
-func (t *testTensor) AvgPool1D(ctx ml.Context, k, s, p int) ml.Tensor {
-	panic("not implemented")
-}
-
-func (t *testTensor) AvgPool2D(ctx ml.Context, k, s int, p float32) ml.Tensor {
-	panic("not implemented")
-}
-
 func (t *testTensor) Conv2D(ctx ml.Context, weight ml.Tensor, s0, s1, p0, p1, d0, d1 int) ml.Tensor {
 	panic("not implemented")
 }
 
-func (t *testTensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, dim, ropeType uint32, base, scale float32) ml.Tensor {
+func (t *testTensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, dim uint32, base, scale float32) ml.Tensor {
 	panic("not implemented")
 }
 
@@ -503,10 +495,6 @@ func (t *testTensor) Contiguous(ctx ml.Context) ml.Tensor {
 	panic("not implemented")
 }
 
-func (t *testTensor) Set(ctx ml.Context, t2 ml.Tensor, offset int, strides ...int) ml.Tensor {
-	panic("not implemented")
-}
-
 func (t *testTensor) Pad(ctx ml.Context, shape ...int) ml.Tensor {
 	panic("not implemented")
 }

llama/patches/0020-ollama-debug-tensor.patch

@@ -1,33 +0,0 @@
-From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
-From: Michael Yang <mxyng@pm.me>
-Date: Sun, 9 Mar 2025 14:44:16 -0700
-Subject: [PATCH] ollama debug tensor
-
----
- ggml/src/ggml-cpu/ggml-cpu.c | 6 ++++++
- 1 file changed, 6 insertions(+)
-
-diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
-index 2f606d82..ec60e8fc 100644
---- a/ggml/src/ggml-cpu/ggml-cpu.c
-+++ b/ggml/src/ggml-cpu/ggml-cpu.c
-@@ -11,6 +11,8 @@
- #include "ggml-threading.h"
- #include "ggml.h"
- 
-+#include "ollama-debug.h"
-+
- #if defined(_MSC_VER) || defined(__MINGW32__)
- #include <malloc.h> // using malloc.h with MSC/MINGW
- #elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
-@@ -14103,6 +14105,10 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
- 
-         ggml_compute_forward(&params, node);
- 
-+#ifdef OLLAMA_DEBUG
-+        ollama_debug(node, true);
-+#endif
-+
-         if (state->ith == 0 && cplan->abort_callback &&
-                 cplan->abort_callback(cplan->abort_callback_data)) {
-             atomic_store_explicit(&tp->abort, node_n + 1, memory_order_relaxed);

llm/memory.go

@@ -218,8 +218,8 @@ func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []strin
 		if blk, ok := layers[fmt.Sprintf("blk.%d", i)]; ok {
 			layerSize = blk.Size()
 			layerSize += kv / f.KV().BlockCount()
-			memoryWeights += blk.Size()
 		}
+		memoryWeights += layerSize
 
 		if opts.NumGPU >= 0 && layerCount >= opts.NumGPU {
 			// Stop allocating on GPU(s) once we hit the users target NumGPU
@@ -376,7 +376,7 @@ func (m MemoryEstimate) LogValue() slog.Value {
 				// memory of the weights
 				"total", format.HumanBytes2(m.memoryWeights),
 				// memory of repeating layers
-				"repeating", format.HumanBytes2(m.memoryWeights),
+				"repeating", format.HumanBytes2(m.memoryWeights-m.memoryLayerOutput),
 				// memory of non-repeating layers
 				"nonrepeating", format.HumanBytes2(m.memoryLayerOutput),
 			),

llm/server.go

@@ -271,7 +271,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, a
 
 	var llamaModel *llama.Model
 	var textProcessor model.TextProcessor
-	if envconfig.NewEngine() || f.KV().OllamaEngineRequired() {
+	if envconfig.NewEngine() {
 		textProcessor, err = model.NewTextProcessor(modelPath)
 		if err != nil {
 			// To prepare for opt-out mode, instead of treating this as an error, we fallback to the old runner

ml/backend.go

@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"os"
-	"slices"
 	"strconv"
 	"strings"
 )
@@ -19,7 +18,6 @@ type Config interface {
 
 	Strings(string, ...[]string) []string
 	Uints(string, ...[]uint32) []uint32
-	Floats(string, ...[]float32) []float32
 }
 
 type Backend interface {
@@ -135,10 +133,8 @@ type Tensor interface {
 	RMSNorm(ctx Context, weight Tensor, eps float32) Tensor
 	Scale(ctx Context, s float64) Tensor
 
-	AvgPool2D(ctx Context, k, s int, p float32) Tensor
 	Conv2D(ctx Context, weight Tensor, s0, s1, p0, p1, d0, d1 int) Tensor
-
-	RoPE(ctx Context, positionIDs, ropeFactors Tensor, dim, ropeType uint32, base, scale float32) Tensor
+	RoPE(ctx Context, positionIDs, ropeFactors Tensor, dim uint32, base, scale float32) Tensor
 
 	Tanh(ctx Context) Tensor
 	GELU(ctx Context) Tensor
@@ -148,7 +144,6 @@ type Tensor interface {
 	View(ctx Context, offset int, shape ...int) Tensor
 	Permute(ctx Context, shape ...int) Tensor
 	Contiguous(ctx Context) Tensor
-	Set(ctx Context, t2 Tensor, offset int, strides ...int) Tensor
 
 	Pad(ctx Context, shape ...int) Tensor
 	Unpad(ctx Context, shape ...int) Tensor
@@ -246,17 +241,16 @@ func dump[S ~[]E, E number](ctx Context, t Tensor, items int, fn func(E) string)
 	}
 
 	shape := t.Shape()
-	slices.Reverse(shape)
 
 	var sb strings.Builder
 	var f func([]int, int)
 	f = func(dims []int, stride int) {
 		prefix := strings.Repeat(" ", len(shape)-len(dims)+1)
-		sb.WriteString("[")
-		defer func() { sb.WriteString("]") }()
+		fmt.Fprint(&sb, "[")
+		defer func() { fmt.Fprint(&sb, "]") }()
 		for i := 0; i < dims[0]; i++ {
 			if i >= items && i < dims[0]-items {
-				sb.WriteString("..., ")
+				fmt.Fprint(&sb, "..., ")
 				// skip to next printable element
 				skip := dims[0] - 2*items
 				if len(dims) > 1 {
@@ -271,14 +265,9 @@ func dump[S ~[]E, E number](ctx Context, t Tensor, items int, fn func(E) string)
 					fmt.Fprint(&sb, ",", strings.Repeat("\n", len(dims)-1), prefix)
 				}
 			} else {
-				text := fn(s[stride+i])
-				if len(text) > 0 && text[0] != '-' {
-					sb.WriteString(" ")
-				}
-
-				sb.WriteString(text)
+				fmt.Fprint(&sb, fn(s[stride+i]))
 				if i < dims[0]-1 {
-					sb.WriteString(", ")
+					fmt.Fprint(&sb, ", ")
 				}
 			}
 		}

ml/backend/ggml/ggml.go

@@ -240,22 +240,11 @@ func New(r *os.File, params ml.BackendParams) (ml.Backend, error) {
 		switch {
 		case contains(t.Name, "position_embd", "token_embd", "token_norm_embd", "token_types"):
 			createTensor(tensor{source: t}, input.bts)
-			if _, ok := meta.Tensors().GroupLayers()["output"]; !ok && t.Name == "token_embd.weight" {
-				createTensor(tensor{source: t, target: "output.weight"}, output.bts)
-			}
 		case contains(t.Name, "cls", "output", "output_norm"):
 			createTensor(tensor{source: t}, output.bts)
 		case strings.HasPrefix(t.Name, "v.") || strings.HasPrefix(t.Name, "mm."):
 			// TODO: assign vision tensors to the gpu if possible
-			createTensor(tensor{source: t}, output.bts)
-		case contains(t.Name, "rope_freqs", "rope_factors_long", "rope_factors_short"):
-			// these tensors should be repeated per layer
-			for i, layer := range layers {
-				createTensor(tensor{
-					source: t,
-					target: "blk." + strconv.Itoa(i) + "." + t.Name,
-				}, layer.bts)
-			}
+			createTensor(tensor{source: t}, input.bts)
 		default:
 			layerIndex := -1
 			if fields := strings.FieldsFunc(t.Name, func(r rune) bool { return !unicode.IsNumber(r) }); len(fields) > 0 {
@@ -267,8 +256,14 @@ func New(r *os.File, params ml.BackendParams) (ml.Backend, error) {
 			if layerIndex >= 0 {
 				createTensor(tensor{source: t}, layers[layerIndex].bts)
 			} else {
-				// load all other tensors on the cpu
-				createTensor(tensor{source: t}, input.bts)
+				// this is a repeating tensor that doesn't explicitly associated with a layer so
+				// duplicate it for each layer
+				for i, layer := range layers {
+					createTensor(tensor{
+						source: t,
+						target: "blk." + strconv.Itoa(i) + "." + t.Name,
+					}, layer.bts)
+				}
 			}
 		}
 	}
@@ -357,7 +352,7 @@ func New(r *os.File, params ml.BackendParams) (ml.Backend, error) {
 
 		if C.ggml_backend_is_cpu(b) {
 			// set number of threads for cpu backend
-			C.ggml_backend_cpu_set_n_threads(b, C.int(Threads(params.NumThreads)))
+			C.ggml_backend_cpu_set_n_threads(b, C.int(params.NumThreads))
 		}
 	}
 
@@ -898,13 +893,10 @@ func (t *Tensor) View(ctx ml.Context, offset int, shape ...int) ml.Tensor {
 }
 
 const (
-	ropeTypeNorm   C.int = 0
-	ropeTypeNeox   C.int = 2
-	ropeTypeMrope  C.int = 8
-	ropeTypeVision C.int = 24
+	ropeTypeNorm C.int = iota
 )
 
-func (t *Tensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, ropeDim, ropeType uint32, ropeBase, ropeScale float32) ml.Tensor {
+func (t *Tensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, ropeDim uint32, ropeBase, ropeScale float32) ml.Tensor {
 	if ropeFactors == nil {
 		ropeFactors = &Tensor{b: t.b}
 	}
@@ -919,8 +911,8 @@ func (t *Tensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, ropeDi
 		t: C.ggml_rope_ext(
 			ctx.(*Context).ctx, dequant, positionIDs.(*Tensor).t, ropeFactors.(*Tensor).t,
 			C.int(ropeDim),
-			C.int(ropeType),
-			131072, // YaRN n_ctx_train
+			131072,       // YaRN n_ctx_train
+			ropeTypeNorm, // ROPE_TYPE_NORM
 			C.float(ropeBase),
 			C.float(ropeScale),
 			0.,  // YaRN ext_factor
@@ -952,27 +944,6 @@ func (t *Tensor) Conv2D(ctx ml.Context, t2 ml.Tensor, s0, s1, p0, p1, d0, d1 int
 	}
 }
 
-func (t *Tensor) AvgPool2D(ctx ml.Context, k, s int, p float32) ml.Tensor {
-	return &Tensor{
-		b: t.b,
-		t: C.ggml_pool_2d(ctx.(*Context).ctx, t.t, C.GGML_OP_POOL_AVG, C.int(k), C.int(k), C.int(s), C.int(s), C.float(p), C.float(p)),
-	}
-}
-
-func (t *Tensor) Set(ctx ml.Context, t2 ml.Tensor, offset int, strides ...int) ml.Tensor {
-	var tt *C.struct_ggml_tensor
-	switch len(strides) {
-	case 0:
-		tt = C.ggml_set_1d(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.size_t(offset))
-	case 1:
-		tt = C.ggml_set_2d(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.size_t(offset), C.size_t(strides[0]))
-	default:
-		panic("unsupported number of dimensions")
-	}
-
-	return &Tensor{b: t.b, t: tt}
-}
-
 func (t *Tensor) ScaledDotProductAttention(ctx ml.Context, key, value, mask ml.Tensor, scale float64) ml.Tensor {
 	var kqMask *C.struct_ggml_tensor
 	if mask != nil {

ml/backend/ggml/ggml/include/ollama-debug.h

@@ -1,11 +0,0 @@
-#include "ggml.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void ollama_debug(const struct ggml_tensor *tensor, bool verbose);
-
-#ifdef __cplusplus
-}
-#endif

ml/backend/ggml/ggml/src/ggml-cpu/cpu_debug.go

@@ -1,6 +0,0 @@
-//go:build debug
-
-package cpu
-
-// #cgo CPPFLAGS: -DOLLAMA_DEBUG
-import "C"

ml/backend/ggml/ggml/src/ggml-cpu/ggml-cpu.c

@@ -11,8 +11,6 @@
 #include "ggml-threading.h"
 #include "ggml.h"
 
-#include "ollama-debug.h"
-
 #if defined(_MSC_VER) || defined(__MINGW32__)
 #include <malloc.h> // using malloc.h with MSC/MINGW
 #elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
@@ -14105,10 +14103,6 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
 
         ggml_compute_forward(&params, node);
 
-#ifdef OLLAMA_DEBUG
-        ollama_debug(node, true);
-#endif
-
         if (state->ith == 0 && cplan->abort_callback &&
                 cplan->abort_callback(cplan->abort_callback_data)) {
             atomic_store_explicit(&tp->abort, node_n + 1, memory_order_relaxed);

ml/backend/ggml/ggml/src/ollama-debug.c

@@ -1,116 +0,0 @@
-#include <string.h>
-#include <inttypes.h>
-
-#include "ollama-debug.h"
-
-static int mul(int64_t *dims, int ndims) {
-    int result = 1;
-    for (int i = 0; i < ndims; i++) {
-        result *= dims[i];
-    }
-
-    return result;
-}
-
-static void repeat(char c, int n) {
-    for (int i = 0; i < n; i++) {
-        fprintf(stderr, "%c", c);
-    }
-}
-
-static void print_tensor(const void *tensor, void (*cb)(const void *, int),
-                         int shape,
-                         int64_t *dims, int ndims, int stride,
-                         int nitems, int pad) {
-    fprintf(stderr, "[");
-    for (int i = 0; i < dims[0]; i++) {
-        if (i >= nitems && i < dims[0] - nitems) {
-            fprintf(stderr, "... (%" PRIi64 " more), ", dims[0] - 2 * nitems);
-            int skip = dims[0] - 2 * nitems;
-            if (ndims > 1) {
-                stride += mul(dims + 1, ndims - 1) * skip;
-                repeat('\n', ndims - 1);
-                repeat(' ', shape - ndims + 1 + pad);
-            }
-            i += skip - 1;
-        } else if (ndims > 1) {
-            print_tensor(tensor, cb, shape, dims + 1, ndims - 1, stride,
-                         nitems, pad);
-            stride += mul(dims + 1, ndims - 1);
-            if (i < dims[0] - 1) {
-                fprintf(stderr, ", ");
-                repeat('\n', ndims - 1);
-                repeat(' ', shape - ndims + 1 + pad);
-            }
-        } else {
-            cb(tensor, stride + i);
-            if (i < dims[0] - 1) {
-                fprintf(stderr, ", ");
-            }
-        }
-    }
-    fprintf(stderr, "]");
-}
-
-static void print_tensor_f16(const void *tensor, int i) {
-    float value = ggml_fp16_to_fp32(((const ggml_fp16_t *)tensor)[i]);
-    fprintf(stderr, "%s%f", value < 0 ? "" : " ", value);
-}
-
-static void print_tensor_f32(const void *tensor, int i) {
-    float value = ((const float *)tensor)[i];
-    fprintf(stderr, "%s%f", value < 0 ? "" : " ", value);
-}
-
-static void print_tensor_i32(const void *tensor, int i) {
-    int32_t value = ((const int32_t *)tensor)[i];
-    fprintf(stderr, "%s%d", value < 0 ? "" : " ", value);
-}
-
-static void ollama_debug_tensor(const struct ggml_tensor *tensor, bool verbose, const char *prefix, int indent) {
-    fprintf(stderr, "%s%s %s (%s): [%" PRIi64 " %" PRIi64 " %" PRIi64 " %" PRIi64 "]\n", prefix, tensor->name,
-            ggml_op_name(tensor->op), ggml_type_name(tensor->type), tensor->ne[0],
-            tensor->ne[1], tensor->ne[2], tensor->ne[3]);
-
-    if (!verbose) {
-        return;
-    }
-
-    for (int i = 0; i < indent; i++) {
-        fprintf(stderr, " ");
-    }
-
-    switch (tensor->type) {
-    case GGML_TYPE_F16:
-        print_tensor(ggml_get_data(tensor), print_tensor_f16, ggml_n_dims(tensor),
-                     (int64_t *)tensor->ne, ggml_n_dims(tensor), 0, 3, indent);
-        break;
-    case GGML_TYPE_F32:
-        print_tensor(ggml_get_data(tensor), print_tensor_f32, ggml_n_dims(tensor),
-                     (int64_t *)tensor->ne, ggml_n_dims(tensor), 0, 3, indent);
-        break;
-    case GGML_TYPE_I32:
-        print_tensor(ggml_get_data(tensor), print_tensor_i32, ggml_n_dims(tensor),
-                     (int64_t *)tensor->ne, ggml_n_dims(tensor), 0, 3, indent);
-        break;
-    default:
-        fprintf(stderr, "<unsupported type>\n");
-        return;
-    }
-
-    fprintf(stderr, "\n");
-}
-
-void ollama_debug(const struct ggml_tensor *tensor, bool verbose) {
-    ollama_debug_tensor(tensor, verbose, ">>> ", 4);
-
-    for (int i = 0; i < GGML_MAX_SRC && tensor->src[i] != NULL; ++i) {
-        char src[8];
-        const int n = snprintf(src, sizeof(src), " src%d ", i);
-        if (n >= sizeof(src)) {
-            src[sizeof(src) - 1] = '\0';
-        }
-
-        ollama_debug_tensor(tensor->src[i], verbose, src, 4);
-    }
-}

ml/backend/ggml/threads.go

@@ -1,7 +0,0 @@
-//go:build !debug
-
-package ggml
-
-func Threads(n int) int {
-	return n
-}

ml/backend/ggml/threads_debug.go

@@ -1,7 +0,0 @@
-//go:build debug
-
-package ggml
-
-func Threads(_ int) int {
-	return 1
-}

model/model.go

@@ -22,8 +22,6 @@ import (
 	"github.com/ollama/ollama/model/input"
 )
 
-var ErrNoVisionModel = errors.New("this model is missing data required for image input")
-
 // Model implements a specific model architecture, defining the forward pass and any model-specific configuration
 type Model interface {
 	Forward(ml.Context, input.Options) (ml.Tensor, error)

model/models/gemma2/model.go

@@ -1,220 +0,0 @@
-package gemma2
-
-import (
-	"math"
-
-	"github.com/ollama/ollama/kvcache"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-	"github.com/ollama/ollama/model"
-	"github.com/ollama/ollama/model/input"
-)
-
-type Options struct {
-	hiddenSize, numHeads, numKVHeads int
-	attnKeyLen, attnValLen           int
-	eps, ropeBase, ropeScale         float32
-	attnLogitSoftcap                 float32
-	finalLogitSoftcap                float32
-	largeModelScaling                bool
-}
-
-type Model struct {
-	model.Base
-	model.SentencePieceModel
-
-	TokenEmbedding *nn.Embedding `gguf:"token_embd"`
-	Layers         []Layer       `gguf:"blk"`
-	OutputNorm     *nn.RMSNorm   `gguf:"output_norm"`
-	Output         *nn.Linear    `gguf:"output,alt:token_embd"` // just set to token_embd?
-
-	*Options
-}
-
-const (
-	gemma27BLayerCount = 46
-)
-
-func New(c ml.Config) (model.Model, error) {
-	m := Model{
-		SentencePieceModel: model.NewSentencePieceModel(
-			c.String("tokenizer.ggml.pretokenizer", `(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+`),
-			&model.Vocabulary{
-				Values: c.Strings("tokenizer.ggml.tokens"),
-				Scores: c.Floats("tokenizer.ggml.scores"),
-				Types:  c.Uints("tokenizer.ggml.token_type"),
-				BOS:    int32(c.Uint("tokenizer.ggml.bos_token_id")),
-				EOS:    int32(c.Uint("tokenizer.ggml.eos_token_id")),
-			},
-		),
-		Layers: make([]Layer, c.Uint("block_count")),
-		Options: &Options{
-			hiddenSize:        int(c.Uint("embedding_length")),
-			numHeads:          int(c.Uint("attention.head_count")),
-			numKVHeads:        int(c.Uint("attention.head_count_kv")),
-			attnKeyLen:        int(c.Uint("attention.key_length")),
-			attnValLen:        int(c.Uint("attention.value_length")),
-			eps:               c.Float("attention.layer_norm_rms_epsilon"),
-			ropeBase:          c.Float("rope.freq_base", 10000.0),
-			ropeScale:         c.Float("rope.freq_scale", 1.0),
-			attnLogitSoftcap:  c.Float("attn_logit_softcapping"),
-			finalLogitSoftcap: c.Float("final_logit_softcapping"),
-		},
-	}
-
-	slidingWindowLen := int32(c.Uint("attention.sliding_window"))
-	m.Cache = kvcache.NewWrapperCache(kvcache.NewSWACache(slidingWindowLen, m.Shift), kvcache.NewCausalCache(m.Shift))
-	m.Cache.SetConfig(ml.CacheConfig{})
-
-	return &m, nil
-}
-
-type SelfAttention struct {
-	Query  *nn.Linear `gguf:"attn_q"`
-	Key    *nn.Linear `gguf:"attn_k"`
-	Value  *nn.Linear `gguf:"attn_v"`
-	Output *nn.Linear `gguf:"attn_output"`
-}
-
-func (sa *SelfAttention) Forward(ctx ml.Context, hiddenState, positionIDs ml.Tensor, cache kvcache.Cache, opts *Options) ml.Tensor {
-	batchSize := hiddenState.Dim(1)
-	ropeType := uint32(2)
-
-	q := sa.Query.Forward(ctx, hiddenState)
-	q = q.Reshape(ctx, opts.attnKeyLen, opts.numHeads, batchSize)
-	q = q.RoPE(ctx, positionIDs, nil, uint32(opts.attnKeyLen), ropeType, opts.ropeBase, opts.ropeScale)
-
-	if opts.largeModelScaling {
-		q = q.Scale(ctx, 1.0/math.Sqrt(float64(opts.hiddenSize/opts.numHeads)))
-	} else {
-		q = q.Scale(ctx, 1.0/math.Sqrt(float64(opts.attnKeyLen)))
-	}
-
-	k := sa.Key.Forward(ctx, hiddenState)
-	k = k.Reshape(ctx, opts.attnKeyLen, opts.numKVHeads, batchSize)
-	k = k.RoPE(ctx, positionIDs, nil, uint32(opts.attnKeyLen), ropeType, opts.ropeBase, opts.ropeScale)
-
-	v := sa.Value.Forward(ctx, hiddenState)
-	v = v.Reshape(ctx, opts.attnValLen, opts.numKVHeads, batchSize)
-
-	cache.Put(ctx, k, v)
-	k, v, mask := cache.Get(ctx)
-
-	q = q.Permute(ctx, 0, 2, 1, 3)
-	k = k.Permute(ctx, 0, 2, 1, 3)
-	v = v.Permute(ctx, 1, 2, 0, 3).Contiguous(ctx)
-
-	kq := k.Mulmat(ctx, q)
-
-	// logit softcap
-	kq = kq.Scale(ctx, 1.0/float64(opts.attnLogitSoftcap))
-	kq = kq.Tanh(ctx)
-	kq = kq.Scale(ctx, float64(opts.attnLogitSoftcap))
-
-	kq = kq.Add(ctx, mask)
-	kq = kq.Softmax(ctx)
-
-	kqv := v.Mulmat(ctx, kq)
-	kqv = kqv.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
-	kqv = kqv.Reshape(ctx, opts.attnValLen*opts.numHeads, batchSize)
-
-	return sa.Output.Forward(ctx, kqv)
-}
-
-func (m *Model) Shift(ctx ml.Context, layer int, key, shift ml.Tensor) (ml.Tensor, error) {
-	return key.RoPE(ctx, shift, nil, uint32(m.Options.attnKeyLen), uint32(2), m.Options.ropeBase, m.Options.ropeScale), nil
-}
-
-type MLP struct {
-	Up   *nn.Linear `gguf:"ffn_up"`
-	Down *nn.Linear `gguf:"ffn_down"`
-	Gate *nn.Linear `gguf:"ffn_gate"`
-}
-
-func (mlp *MLP) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *Options) ml.Tensor {
-	hiddenState = mlp.Gate.Forward(ctx, hiddenState).GELU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenState))
-	return mlp.Down.Forward(ctx, hiddenState)
-}
-
-type Layer struct {
-	AttentionNorm     *nn.RMSNorm `gguf:"attn_norm"`
-	SelfAttention     *SelfAttention
-	PostAttentionNorm *nn.RMSNorm `gguf:"post_attention_norm"`
-	MLPNorm           *nn.RMSNorm `gguf:"ffn_norm"`
-	MLP               *MLP
-	PostMLPNorm       *nn.RMSNorm `gguf:"post_ffw_norm"`
-}
-
-func (l *Layer) Forward(ctx ml.Context, hiddenState, positionIDs, outputs ml.Tensor, cache kvcache.Cache, opts *Options) ml.Tensor {
-	residual := hiddenState
-
-	hiddenState = l.AttentionNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = l.SelfAttention.Forward(ctx, hiddenState, positionIDs, cache, opts)
-	hiddenState = l.PostAttentionNorm.Forward(ctx, hiddenState, opts.eps)
-
-	// In the final layer (outputs != nil), optimize by pruning to just the token positions
-	// we need logits for.
-	if outputs != nil {
-		hiddenState = hiddenState.Rows(ctx, outputs)
-		residual = residual.Rows(ctx, outputs)
-	}
-
-	hiddenState = hiddenState.Add(ctx, residual)
-	residual = hiddenState
-
-	hiddenState = l.MLPNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = l.MLP.Forward(ctx, hiddenState, opts)
-	hiddenState = l.PostMLPNorm.Forward(ctx, hiddenState, opts.eps)
-	return hiddenState.Add(ctx, residual)
-}
-
-func (m *Model) Forward(ctx ml.Context, opts input.Options) (ml.Tensor, error) {
-	inputs, err := ctx.Input().FromIntSlice(opts.Inputs, len(opts.Inputs))
-	if err != nil {
-		return nil, err
-	}
-
-	positions, err := ctx.Input().FromIntSlice(opts.Positions, len(opts.Positions))
-	if err != nil {
-		return nil, err
-	}
-
-	outputs, err := ctx.Output().FromIntSlice(opts.Outputs, len(opts.Outputs))
-	if err != nil {
-		return nil, err
-	}
-
-	hiddenState := m.TokenEmbedding.Forward(ctx, inputs)
-	hiddenState = hiddenState.Scale(ctx, math.Sqrt(float64(m.Options.hiddenSize)))
-
-	if len(m.Layers) == gemma27BLayerCount {
-		m.Options.largeModelScaling = true
-	}
-
-	for i, layer := range m.Layers {
-		cacheType := i % 2
-		m.Cache.SetLayer(i)
-		wc := m.Cache.(*kvcache.WrapperCache)
-		wc.SetLayerType(cacheType)
-
-		var lastLayerOutputs ml.Tensor
-		if i == len(m.Layers)-1 {
-			lastLayerOutputs = outputs
-		}
-
-		hiddenState = layer.Forward(ctx, hiddenState, positions, lastLayerOutputs, m.Cache, m.Options)
-	}
-
-	hiddenState = m.OutputNorm.Forward(ctx, hiddenState, m.eps)
-	hiddenState = m.Output.Forward(ctx, hiddenState)
-
-	// final logit softcap
-	hiddenState = hiddenState.Scale(ctx, 1.0/float64(m.Options.finalLogitSoftcap))
-	hiddenState = hiddenState.Tanh(ctx)
-	hiddenState = hiddenState.Scale(ctx, float64(m.Options.finalLogitSoftcap))
-	return hiddenState.Rows(ctx, outputs), nil
-}
-
-func init() {
-	model.Register("gemma2", New)
-}

model/models/gemma3/model.go

@@ -1,177 +0,0 @@
-package gemma3
-
-import (
-	"bytes"
-	"encoding/binary"
-	"hash/fnv"
-	"image"
-	"math"
-
-	"github.com/ollama/ollama/kvcache"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-	"github.com/ollama/ollama/model"
-	"github.com/ollama/ollama/model/input"
-)
-
-type Model struct {
-	model.Base
-	model.SentencePieceModel
-
-	*VisionModel `gguf:"v,vision"`
-	*TextModel
-
-	*MultiModalProjector `gguf:"mm"`
-
-	ImageProcessor
-}
-
-var _ model.MultimodalProcessor = (*Model)(nil)
-
-type MultiModalProjector struct {
-	SoftEmbNorm     *nn.RMSNorm `gguf:"mm_soft_emb_norm"`
-	InputProjection *nn.Linear  `gguf:"mm_input_projection"`
-
-	tokensPerImage int
-}
-
-func (p *MultiModalProjector) Forward(ctx ml.Context, visionOutputs ml.Tensor, imageSize, patchSize int, eps float32) ml.Tensor {
-	l := visionOutputs.Dim(0)
-
-	visionOutputs = visionOutputs.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)
-	patchesPerImage := imageSize / patchSize
-	visionOutputs = visionOutputs.Reshape(ctx, patchesPerImage, patchesPerImage, l)
-
-	kernelSize := patchesPerImage / int(math.Sqrt(float64(p.tokensPerImage)))
-	visionOutputs = visionOutputs.AvgPool2D(ctx, kernelSize, kernelSize, 0)
-	visionOutputs = visionOutputs.Reshape(ctx, visionOutputs.Dim(0)*visionOutputs.Dim(1), l)
-	visionOutputs = visionOutputs.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)
-	visionOutputs = p.SoftEmbNorm.Forward(ctx, visionOutputs, eps)
-
-	// TODO: inputProjection must be transposed since they're incompatible with visionOutputs
-	visionOutputs = p.InputProjection.Weight.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx).Mulmat(ctx, visionOutputs)
-	return visionOutputs
-}
-
-func New(c ml.Config) (model.Model, error) {
-	m := Model{
-		SentencePieceModel: model.NewSentencePieceModel(
-			c.String("tokenizer.ggml.pretokenizer", `(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+`),
-			&model.Vocabulary{
-				Values: c.Strings("tokenizer.ggml.tokens"),
-				Scores: c.Floats("tokenizer.ggml.scores"),
-				Types:  c.Uints("tokenizer.ggml.token_type"),
-				BOS:    int32(c.Uint("tokenizer.ggml.bos_token_id")),
-				AddBOS: c.Bool("tokenizer.ggml.add_bos_token", true),
-				EOS:    int32(1),
-				AddEOS: c.Bool("tokenizer.ggml.add_eos_token", false),
-				EOT:    int32(106),
-				AddEOT: c.Bool("tokenizer.ggml.add_eot_token", false),
-			},
-		),
-		ImageProcessor: newImageProcessor(c),
-		VisionModel:    newVisionModel(c),
-		TextModel:      newTextModel(c),
-		MultiModalProjector: &MultiModalProjector{
-			tokensPerImage: int(c.Uint("mm_tokens_per_image", 256)),
-		},
-	}
-
-	slidingWindowLen := int32(c.Uint("attention.sliding_window"))
-	m.Cache = kvcache.NewWrapperCache(kvcache.NewSWACache(slidingWindowLen, m.Shift), kvcache.NewCausalCache(m.Shift))
-
-	return &m, nil
-}
-
-func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) (any, error) {
-	if len(m.VisionModel.Layers) == 0 {
-		return nil, model.ErrNoVisionModel
-	}
-
-	image, _, err := image.Decode(bytes.NewReader(multimodalData))
-	if err != nil {
-		return nil, err
-	}
-
-	f32s, err := m.ImageProcessor.ProcessImage(image)
-	if err != nil {
-		return nil, err
-	}
-
-	pixelValues, err := ctx.Input().FromFloatSlice(f32s,
-		m.ImageProcessor.imageSize,
-		m.ImageProcessor.imageSize,
-		m.ImageProcessor.numChannels,
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	visionOutputs := m.VisionModel.Forward(ctx, pixelValues)
-	visionOutputs = m.MultiModalProjector.Forward(ctx, visionOutputs, m.imageSize, m.patchSize, m.VisionModel.eps)
-	return visionOutputs, nil
-}
-
-type imageToken struct {
-	embedding ml.Tensor
-	index     int
-}
-
-func (m *Model) PostTokenize(ctx ml.Context, inputs []input.Input) ([]input.Input, error) {
-	var result []input.Input
-	fnvHash := fnv.New64a()
-
-	for _, inp := range inputs {
-		if inp.Multimodal == nil {
-			result = append(result, inp)
-		} else {
-			imageInputs := []input.Input{
-				{Token: 108},    // "\n\n"
-				{Token: 255999}, // "<start_of_image>""
-			}
-			result = append(result, imageInputs...)
-
-			// add image embeddings
-			inputMultimodal := inp.Multimodal.(ml.Tensor)
-
-			for i := range inputMultimodal.Dim(1) {
-				fnvHash.Reset()
-				binary.Write(fnvHash, binary.NativeEndian, inp.MultimodalHash)
-				fnvHash.Write([]byte{byte(i)})
-
-				imageToken := imageToken{embedding: inputMultimodal, index: i}
-				result = append(result, input.Input{Multimodal: imageToken, MultimodalHash: fnvHash.Sum64()})
-			}
-
-			result = append(result,
-				input.Input{Token: 256000}, // <end_of_image>
-				input.Input{Token: 108},    // "\n\n"
-			)
-		}
-	}
-
-	return result, nil
-}
-
-func (m *Model) Forward(ctx ml.Context, opts input.Options) (ml.Tensor, error) {
-	inputs, err := ctx.Input().FromIntSlice(opts.Inputs, len(opts.Inputs))
-	if err != nil {
-		return nil, err
-	}
-
-	positions, err := ctx.Input().FromIntSlice(opts.Positions, len(opts.Positions))
-	if err != nil {
-		return nil, err
-	}
-
-	outputs, err := ctx.Output().FromIntSlice(opts.Outputs, len(opts.Outputs))
-	if err != nil {
-		return nil, err
-	}
-
-	return m.TextModel.Forward(ctx, inputs, positions, outputs, opts, m.Cache), nil
-}
-
-func init() {
-	model.Register("gemma3", New)
-}

model/models/gemma3/model_text.go

@@ -1,247 +0,0 @@
-package gemma3
-
-import (
-	"math"
-
-	"github.com/ollama/ollama/kvcache"
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-	"github.com/ollama/ollama/model"
-	"github.com/ollama/ollama/model/input"
-)
-
-type TextOptions struct {
-	hiddenSize, numHeads, numKVHeads int
-	attnKeyLen, attnValLen           int
-	eps, ropeScale                   float32
-	ropeLocalBase, ropeGlobalBase    float32
-	largeModelScaling                bool
-}
-
-type TextModel struct {
-	model.Base
-	model.SentencePieceModel
-
-	TokenEmbedding *nn.Embedding `gguf:"token_embd"`
-	Layers         []TextLayer   `gguf:"blk"`
-	OutputNorm     *nn.RMSNorm   `gguf:"output_norm"`
-	Output         *nn.Linear    `gguf:"output,alt:token_embd"`
-
-	*TextOptions
-}
-
-const (
-	gemmaGlobalCacheCount = 6
-	gemma27BLayerCount    = 62
-)
-
-const (
-	cacheTypeSWA = iota
-	cacheTypeCausal
-)
-
-func newTextModel(c ml.Config) *TextModel {
-	numBlocks := int(c.Uint("block_count"))
-
-	m := TextModel{
-		SentencePieceModel: model.NewSentencePieceModel(
-			c.String("tokenizer.ggml.pretokenizer", `(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+`),
-			&model.Vocabulary{
-				Values: c.Strings("tokenizer.ggml.tokens"),
-				Scores: c.Floats("tokenizer.ggml.scores"),
-				Types:  c.Uints("tokenizer.ggml.token_type"),
-				BOS:    int32(c.Uint("tokenizer.ggml.bos_token_id")),
-				EOS:    int32(c.Uint("tokenizer.ggml.eos_token_id")),
-			},
-		),
-		Layers: make([]TextLayer, numBlocks),
-		TextOptions: &TextOptions{
-			hiddenSize:     int(c.Uint("embedding_length")),
-			numHeads:       int(c.Uint("attention.head_count")),
-			numKVHeads:     int(c.Uint("attention.head_count_kv")),
-			attnKeyLen:     int(c.Uint("attention.key_length", 256)),
-			attnValLen:     int(c.Uint("attention.value_length", 256)),
-			eps:            c.Float("attention.layer_norm_rms_epsilon", 1e-06),
-			ropeLocalBase:  c.Float("rope.local.freq_base", 10000.0),
-			ropeGlobalBase: c.Float("rope.global.freq_base", 1000000.0),
-			ropeScale:      c.Float("rope.freq_scale", 1.0),
-		},
-	}
-
-	if numBlocks == gemma27BLayerCount {
-		m.largeModelScaling = true
-	}
-
-	return &m
-}
-
-type TextSelfAttention struct {
-	Query     *nn.Linear  `gguf:"attn_q"`
-	QueryNorm *nn.RMSNorm `gguf:"attn_q_norm"`
-	Key       *nn.Linear  `gguf:"attn_k"`
-	KeyNorm   *nn.RMSNorm `gguf:"attn_k_norm"`
-	Value     *nn.Linear  `gguf:"attn_v"`
-	Output    *nn.Linear  `gguf:"attn_output"`
-}
-
-func (sa *TextSelfAttention) Forward(ctx ml.Context, layer int, hiddenState, positionIDs ml.Tensor, cache kvcache.Cache, opts *TextOptions) ml.Tensor {
-	batchSize := hiddenState.Dim(1)
-	ropeType := uint32(2)
-
-	ropeBase := opts.ropeLocalBase
-	if (layer+1)%gemmaGlobalCacheCount == 0 {
-		ropeBase = opts.ropeGlobalBase
-	}
-
-	q := sa.Query.Forward(ctx, hiddenState)
-	q = q.Reshape(ctx, opts.attnKeyLen, opts.numHeads, batchSize)
-	q = sa.QueryNorm.Forward(ctx, q, opts.eps)
-	q = q.RoPE(ctx, positionIDs, nil, uint32(opts.attnKeyLen), ropeType, ropeBase, opts.ropeScale)
-
-	if opts.largeModelScaling {
-		q = q.Scale(ctx, 1.0/math.Sqrt(float64(opts.hiddenSize/opts.numHeads)))
-	} else {
-		q = q.Scale(ctx, 1.0/math.Sqrt(float64(opts.attnKeyLen)))
-	}
-
-	k := sa.Key.Forward(ctx, hiddenState)
-	k = k.Reshape(ctx, opts.attnKeyLen, opts.numKVHeads, batchSize)
-	k = sa.KeyNorm.Forward(ctx, k, opts.eps)
-	k = k.RoPE(ctx, positionIDs, nil, uint32(opts.attnKeyLen), ropeType, ropeBase, opts.ropeScale)
-
-	v := sa.Value.Forward(ctx, hiddenState)
-	v = v.Reshape(ctx, opts.attnValLen, opts.numKVHeads, batchSize)
-
-	scaleFactor := 1.0
-	kqv := nn.Attention(ctx, q, k, v, scaleFactor, cache)
-	kqv = kqv.Reshape(ctx, opts.attnValLen*opts.numHeads, batchSize)
-
-	return sa.Output.Forward(ctx, kqv)
-}
-
-func (m *TextModel) Shift(ctx ml.Context, layer int, key, shift ml.Tensor) (ml.Tensor, error) {
-	ropeBase := m.TextOptions.ropeLocalBase
-	if (layer+1)%gemmaGlobalCacheCount == 0 {
-		ropeBase = m.TextOptions.ropeGlobalBase
-	}
-
-	return key.RoPE(ctx, shift, nil, uint32(m.TextOptions.attnKeyLen), uint32(2), ropeBase, m.TextOptions.ropeScale), nil
-}
-
-type TextMLP struct {
-	Up   *nn.Linear `gguf:"ffn_up"`
-	Down *nn.Linear `gguf:"ffn_down"`
-	Gate *nn.Linear `gguf:"ffn_gate"`
-}
-
-func (mlp *TextMLP) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *TextOptions) ml.Tensor {
-	hiddenState = mlp.Gate.Forward(ctx, hiddenState).GELU(ctx).Mul(ctx, mlp.Up.Forward(ctx, hiddenState))
-	return mlp.Down.Forward(ctx, hiddenState)
-}
-
-type TextLayer struct {
-	AttentionNorm     *nn.RMSNorm `gguf:"attn_norm"`
-	SelfAttention     *TextSelfAttention
-	PostAttentionNorm *nn.RMSNorm `gguf:"post_attention_norm"`
-	MLPNorm           *nn.RMSNorm `gguf:"ffn_norm"`
-	MLP               *TextMLP
-	PostMLPNorm       *nn.RMSNorm `gguf:"post_ffw_norm"`
-}
-
-func (l *TextLayer) Forward(ctx ml.Context, layer int, hiddenState, positionIDs, outputs ml.Tensor, cache kvcache.Cache, opts *TextOptions) ml.Tensor {
-	residual := hiddenState
-
-	hiddenState = l.AttentionNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = l.SelfAttention.Forward(ctx, layer, hiddenState, positionIDs, cache, opts)
-	hiddenState = l.PostAttentionNorm.Forward(ctx, hiddenState, opts.eps)
-
-	// In the final layer (outputs != nil), optimize by pruning to just the token positions
-	// we need logits for.
-	if outputs != nil {
-		hiddenState = hiddenState.Rows(ctx, outputs)
-		residual = residual.Rows(ctx, outputs)
-	}
-
-	hiddenState = hiddenState.Add(ctx, residual)
-	residual = hiddenState
-
-	hiddenState = l.MLPNorm.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = l.MLP.Forward(ctx, hiddenState, opts)
-	hiddenState = l.PostMLPNorm.Forward(ctx, hiddenState, opts.eps)
-	return hiddenState.Add(ctx, residual)
-}
-
-func setImageEmbeddings(ctx ml.Context, hiddenState ml.Tensor, multimodal []input.MultimodalIndex) []int {
-	var embedding ml.Tensor
-	var src, dst, length int
-	var except []int
-
-	for _, image := range multimodal {
-		imageToken := image.Multimodal.(imageToken)
-		imageSrc := imageToken.index
-		imageDst := image.Index
-
-		if embedding == nil {
-			embedding = imageToken.embedding
-			src = imageSrc
-			dst = imageDst
-			length = 1
-		} else if embedding == imageToken.embedding && imageSrc+1 == src && imageDst+1 == dst {
-			src = imageSrc
-			dst = imageDst
-			length++
-		} else if embedding == imageToken.embedding && src+length == imageSrc && dst+length == imageDst {
-			length++
-		} else {
-			visionOutputs := embedding.View(ctx, src*embedding.Stride(1), length*embedding.Dim(0))
-			ctx.Forward(visionOutputs.Copy(ctx, hiddenState.View(ctx, dst*hiddenState.Stride(1), length*hiddenState.Dim(0))))
-
-			embedding = imageToken.embedding
-			src = imageSrc
-			dst = imageDst
-			length = 1
-		}
-
-		except = append(except, imageDst)
-	}
-
-	if embedding != nil {
-		visionOutputs := embedding.View(ctx, src*embedding.Stride(1), length*embedding.Dim(0))
-		ctx.Forward(visionOutputs.Copy(ctx, hiddenState.View(ctx, dst*hiddenState.Stride(1), length*hiddenState.Dim(0))))
-	}
-
-	return except
-}
-
-func (m *TextModel) Forward(ctx ml.Context, inputs, positions, outputs ml.Tensor, opts input.Options, cache kvcache.Cache) ml.Tensor {
-	hiddenState := m.TokenEmbedding.Forward(ctx, inputs)
-	hiddenState = hiddenState.Scale(ctx, math.Sqrt(float64(m.TextOptions.hiddenSize)))
-
-	except := setImageEmbeddings(ctx, hiddenState, opts.Multimodal)
-
-	for i, layer := range m.Layers {
-		// gemma alternates between the sliding window (local) and causal (global)
-		// kv cache every 6 layers
-		cacheType := cacheTypeSWA
-		if (i+1)%gemmaGlobalCacheCount == 0 {
-			cacheType = cacheTypeCausal
-		}
-		cache.SetLayer(i)
-		wc := cache.(*kvcache.WrapperCache)
-		wc.SetLayerType(cacheType)
-
-		if causal, ok := wc.UnderlyingCache().(*kvcache.Causal); ok {
-			causal.SetCausal(ctx, kvcache.CausalOptions{Except: except})
-		}
-
-		var lastLayerOutputs ml.Tensor
-		if i == len(m.Layers)-1 {
-			lastLayerOutputs = outputs
-		}
-
-		hiddenState = layer.Forward(ctx, i, hiddenState, positions, lastLayerOutputs, cache, m.TextOptions)
-	}
-
-	hiddenState = m.OutputNorm.Forward(ctx, hiddenState, m.eps)
-	return m.Output.Forward(ctx, hiddenState)
-}

model/models/gemma3/model_vision.go

@@ -1,127 +0,0 @@
-package gemma3
-
-import (
-	"math"
-
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/ml/nn"
-)
-
-var batchSize int = 1
-
-type VisionSelfAttention struct {
-	Query  *nn.Linear `gguf:"attn_q"`
-	Key    *nn.Linear `gguf:"attn_k"`
-	Value  *nn.Linear `gguf:"attn_v"`
-	Output *nn.Linear `gguf:"attn_output"`
-}
-
-func (sa *VisionSelfAttention) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *VisionModelOptions) ml.Tensor {
-	headDim := opts.hiddenSize / opts.numHeads
-
-	query := sa.Query.Forward(ctx, hiddenState)
-	key := sa.Key.Forward(ctx, hiddenState)
-	value := sa.Value.Forward(ctx, hiddenState)
-
-	query = query.Reshape(ctx, headDim, opts.numHeads, query.Dim(1), batchSize)
-	key = key.Reshape(ctx, headDim, opts.numHeads, key.Dim(1), batchSize)
-	value = value.Reshape(ctx, headDim, opts.numHeads, value.Dim(1), batchSize)
-
-	attention := nn.Attention(ctx, query, key, value, 1.0/math.Sqrt(float64(headDim)), nil)
-	attention = attention.Reshape(ctx, opts.hiddenSize, attention.Dim(2), batchSize)
-
-	hiddenState = sa.Output.Forward(ctx, attention)
-	return hiddenState
-}
-
-type VisionMLP struct {
-	FC1 *nn.Linear `gguf:"fc1"`
-	FC2 *nn.Linear `gguf:"fc2"`
-}
-
-func (mlp *VisionMLP) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *VisionModelOptions) ml.Tensor {
-	hiddenState = mlp.FC1.Forward(ctx, hiddenState).GELU(ctx)
-	hiddenState = mlp.FC2.Forward(ctx, hiddenState)
-	return hiddenState
-}
-
-type VisionEncoderLayer struct {
-	LayerNorm1    *nn.LayerNorm `gguf:"layer_norm1"`
-	SelfAttention *VisionSelfAttention
-
-	LayerNorm2 *nn.LayerNorm `gguf:"layer_norm2"`
-	MLP        *VisionMLP    `gguf:"mlp"`
-}
-
-func (e *VisionEncoderLayer) Forward(ctx ml.Context, hiddenState ml.Tensor, opts *VisionModelOptions) ml.Tensor {
-	residual := hiddenState
-
-	// self attention
-	hiddenState = e.LayerNorm1.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = e.SelfAttention.Forward(ctx, hiddenState, opts)
-	hiddenState = hiddenState.Add(ctx, residual)
-	residual = hiddenState
-
-	// feed forward
-	hiddenState = e.LayerNorm2.Forward(ctx, hiddenState, opts.eps)
-	hiddenState = e.MLP.Forward(ctx, hiddenState, opts)
-	return hiddenState.Add(ctx, residual)
-}
-
-type VisionModelOptions struct {
-	hiddenSize, numHeads int
-	imageSize, patchSize int
-	eps                  float32
-}
-
-type VisionModel struct {
-	PatchEmbedding    *nn.Conv2D    `gguf:"patch_embedding"`
-	PositionEmbedding *nn.Embedding `gguf:"position_embedding"`
-	PostLayerNorm     *nn.LayerNorm `gguf:"post_layernorm"`
-
-	Layers []VisionEncoderLayer `gguf:"blk"`
-
-	*VisionModelOptions
-}
-
-func (m *VisionModel) Forward(ctx ml.Context, pixelValues ml.Tensor) ml.Tensor {
-	numPatches := (m.imageSize / m.patchSize) * (m.imageSize / m.patchSize)
-
-	hiddenState := m.PatchEmbedding.Forward(ctx, pixelValues, m.patchSize, m.patchSize, 0, 0, 1, 1)
-	hiddenState = hiddenState.Reshape(ctx, numPatches, m.hiddenSize)
-	hiddenState = hiddenState.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)
-
-	positions := make([]int32, numPatches)
-	for i := range positions {
-		positions[i] = int32(i)
-	}
-
-	positionIDs, err := ctx.Input().FromIntSlice(positions, len(positions))
-	if err != nil {
-		panic(err)
-	}
-
-	hiddenState = hiddenState.Add(ctx, m.PositionEmbedding.Forward(ctx, positionIDs))
-
-	for _, layer := range m.Layers {
-		hiddenState = layer.Forward(ctx, hiddenState, m.VisionModelOptions)
-	}
-
-	hiddenState = m.PostLayerNorm.Forward(ctx, hiddenState, m.eps)
-	return hiddenState
-}
-
-func newVisionModel(c ml.Config) *VisionModel {
-	return &VisionModel{
-		Layers: make([]VisionEncoderLayer, c.Uint("vision.block_count")),
-		VisionModelOptions: &VisionModelOptions{
-			hiddenSize: int(c.Uint("vision.embedding_length")),
-			numHeads:   int(c.Uint("vision.attention.head_count")),
-
-			imageSize: int(c.Uint("vision.image_size")),
-			patchSize: int(c.Uint("vision.patch_size")),
-
-			eps: c.Float("vision.attention.layer_norm_epsilon"),
-		},
-	}
-}

model/models/gemma3/process_image.go

@@ -1,58 +0,0 @@
-package gemma3
-
-import (
-	"image"
-
-	"github.com/ollama/ollama/ml"
-	"github.com/ollama/ollama/model/imageproc"
-)
-
-type ImageProcessor struct {
-	imageSize, patchSize, numChannels int
-}
-
-func newImageProcessor(c ml.Config) ImageProcessor {
-	return ImageProcessor{
-		imageSize:   int(c.Uint("vision.image_size")),
-		patchSize:   int(c.Uint("vision.patch_size")),
-		numChannels: int(c.Uint("vision.num_channels")),
-	}
-}
-
-func (p *ImageProcessor) pack(img image.Image, mean, std [3]float32) []float32 {
-	var pixelVals, rVals, gVals, bVals []float32
-
-	bounds := img.Bounds()
-	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
-		for x := bounds.Min.X; x < bounds.Max.X; x++ {
-			c := img.At(x, y)
-			r, g, b, _ := c.RGBA()
-			rVal := float32(r>>8) / 255.0
-			gVal := float32(g>>8) / 255.0
-			bVal := float32(b>>8) / 255.0
-
-			rVal = (rVal - mean[0]) / std[0]
-			gVal = (gVal - mean[1]) / std[1]
-			bVal = (bVal - mean[2]) / std[2]
-
-			rVals = append(rVals, rVal)
-			gVals = append(gVals, gVal)
-			bVals = append(bVals, bVal)
-		}
-	}
-
-	pixelVals = append(pixelVals, rVals...)
-	pixelVals = append(pixelVals, gVals...)
-	pixelVals = append(pixelVals, bVals...)
-
-	return pixelVals
-}
-
-func (p ImageProcessor) ProcessImage(img image.Image) ([]float32, error) {
-	outputSize := image.Point{p.imageSize, p.imageSize}
-	newImage := imageproc.Composite(img)
-	newImage = imageproc.Resize(newImage, outputSize, imageproc.ResizeBilinear)
-
-	data := p.pack(newImage, imageproc.ImageNetStandardMean, imageproc.ImageNetStandardSTD)
-	return data, nil
-}

model/models/llama/model.go

@@ -76,15 +76,14 @@ type SelfAttention struct {
 func (sa *SelfAttention) Forward(ctx ml.Context, hiddenState, positionIDs ml.Tensor, cache kvcache.Cache, opts *Options) ml.Tensor {
 	batchSize := hiddenState.Dim(1)
 	headDim := opts.hiddenSize / opts.numHeads
-	ropeType := uint32(0)
 
 	q := sa.Query.Forward(ctx, hiddenState)
 	q = q.Reshape(ctx, headDim, opts.numHeads, batchSize)
-	q = q.RoPE(ctx, positionIDs, sa.RopeFactors, opts.ropeDim, ropeType, opts.ropeBase, opts.ropeScale)
+	q = q.RoPE(ctx, positionIDs, sa.RopeFactors, opts.ropeDim, opts.ropeBase, opts.ropeScale)
 
 	k := sa.Key.Forward(ctx, hiddenState)
 	k = k.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
-	k = k.RoPE(ctx, positionIDs, sa.RopeFactors, opts.ropeDim, ropeType, opts.ropeBase, opts.ropeScale)
+	k = k.RoPE(ctx, positionIDs, sa.RopeFactors, opts.ropeDim, opts.ropeBase, opts.ropeScale)
 
 	v := sa.Value.Forward(ctx, hiddenState)
 	v = v.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
@@ -97,7 +96,7 @@ func (sa *SelfAttention) Forward(ctx ml.Context, hiddenState, positionIDs ml.Ten
 }
 
 func (m *Model) Shift(ctx ml.Context, layer int, key, shift ml.Tensor) (ml.Tensor, error) {
-	return key.RoPE(ctx, shift, m.Layers[layer].SelfAttention.RopeFactors, uint32(0), m.ropeDim, m.ropeBase, m.ropeScale), nil
+	return key.RoPE(ctx, shift, m.Layers[layer].SelfAttention.RopeFactors, m.ropeDim, m.ropeBase, m.ropeScale), nil
 }
 
 type MLP struct {

model/models/mllama/model.go

@@ -63,10 +63,6 @@ func New(c ml.Config) (model.Model, error) {
 }
 
 func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) (any, error) {
-	if len(m.VisionModel.Transformer.Layers) == 0 || len(m.GlobalTransformer.Layers) == 0 {
-		return nil, model.ErrNoVisionModel
-	}
-
 	image, _, err := image.Decode(bytes.NewReader(multimodalData))
 	if err != nil {
 		return nil, err

model/models/mllama/model_text.go

@@ -20,15 +20,14 @@ type TextSelfAttention struct {
 func (sa *TextSelfAttention) Forward(ctx ml.Context, hiddenState, positions, _ ml.Tensor, cache *kvcache.WrapperCache, opts *TextModelOptions) ml.Tensor {
 	batchSize := hiddenState.Dim(1)
 	headDim := opts.hiddenSize / opts.numHeads
-	ropeType := uint32(0)
 
 	query := sa.Query.Forward(ctx, hiddenState)
 	query = query.Reshape(ctx, headDim, opts.numHeads, batchSize)
-	query = query.RoPE(ctx, positions, sa.RopeFactors, opts.ropeDim, ropeType, opts.ropeBase, opts.ropeScale)
+	query = query.RoPE(ctx, positions, sa.RopeFactors, opts.ropeDim, opts.ropeBase, opts.ropeScale)
 
 	key := sa.Key.Forward(ctx, hiddenState)
 	key = key.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
-	key = key.RoPE(ctx, positions, sa.RopeFactors, opts.ropeDim, ropeType, opts.ropeBase, opts.ropeScale)
+	key = key.RoPE(ctx, positions, sa.RopeFactors, opts.ropeDim, opts.ropeBase, opts.ropeScale)
 
 	value := sa.Value.Forward(ctx, hiddenState)
 	value = value.Reshape(ctx, headDim, opts.numKVHeads, batchSize)
@@ -41,9 +40,8 @@ func (sa *TextSelfAttention) Forward(ctx ml.Context, hiddenState, positions, _ m
 }
 
 func (m *TextModel) Shift(ctx ml.Context, layer int, key, shift ml.Tensor) (ml.Tensor, error) {
-	// This will only get called for layers in the cache, which are just the self attention layers
 	if sa, ok := m.Transformer.Layers[layer].(*TextSelfAttentionDecoderLayer); ok {
-		return key.RoPE(ctx, shift, sa.SelfAttention.RopeFactors, m.ropeDim, uint32(0), m.ropeBase, m.ropeScale), nil
+		return key.RoPE(ctx, shift, sa.SelfAttention.RopeFactors, m.ropeDim, m.ropeBase, m.ropeScale), nil
 	}
 
 	return key, nil

model/models/mllama/process_image.go

@@ -144,6 +144,8 @@ func (p *ImageProcessor) splitToTiles(img image.Image, numTilesSize image.Point)
 	return images
 }
 
+// remove the "alpha" channel by drawing over a prefilled image
+//
 // remove the "alpha" channel by drawing over a prefilled image
 //
 //nolint:unused

model/models/models.go

@@ -1,8 +1,6 @@
 package models
 
 import (
-	_ "github.com/ollama/ollama/model/models/gemma2"
-	_ "github.com/ollama/ollama/model/models/gemma3"
 	_ "github.com/ollama/ollama/model/models/llama"
 	_ "github.com/ollama/ollama/model/models/mllama"
 )

model/process_text.go

@@ -4,7 +4,6 @@ import (
 	"cmp"
 	"iter"
 	"log/slog"
-	"slices"
 	"strings"
 	"sync"
 
@@ -19,15 +18,6 @@ const (
 	SpecialEOS
 )
 
-const (
-	TOKEN_TYPE_NORMAL = iota + 1
-	TOKEN_TYPE_UNKNOWN
-	TOKEN_TYPE_CONTROL
-	TOKEN_TYPE_USER_DEFINED
-	TOKEN_TYPE_UNUSED
-	TOKEN_TYPE_BYTE
-)
-
 type TextProcessor interface {
 	Encode(s string, addSpecial bool) ([]int32, error)
 	Decode([]int32) (string, error)
@@ -37,11 +27,11 @@ type TextProcessor interface {
 type Vocabulary struct {
 	Values []string
 	Types  []uint32
-	Scores []float32
+	Scores []uint32
 	Merges []string
 
-	BOS, EOS, EOT          int32
-	AddBOS, AddEOS, AddEOT bool
+	BOS, EOS       int32
+	AddBOS, AddEOS bool
 
 	specialOnce sync.Once
 	special     []string
@@ -58,7 +48,7 @@ func (v *Vocabulary) Is(id int32, special Special) bool {
 	case SpecialBOS:
 		return id == v.BOS
 	case SpecialEOS:
-		return id == v.EOS || id == v.EOT
+		return id == v.EOS
 	default:
 		return false
 	}
@@ -86,9 +76,7 @@ func (v *Vocabulary) Decode(id int32) string {
 func (v *Vocabulary) SpecialVocabulary() []string {
 	v.specialOnce.Do(func() {
 		for i := range v.Values {
-			if slices.Contains([]int{105, 106}, i) {
-				v.special = append(v.special, v.Values[i])
-			} else if v.Types[i] == TOKEN_TYPE_CONTROL {
+			if v.Types[i] == 3 {
 				v.special = append(v.special, v.Values[i])
 			}
 		}

model/process_text_spm.go

@@ -1,246 +0,0 @@
-package model
-
-import (
-	"iter"
-	"log/slog"
-	"strings"
-
-	"github.com/dlclark/regexp2"
-	queue "github.com/emirpasic/gods/v2/queues/priorityqueue"
-)
-
-const spmWhitespaceSep = "▁"
-
-func replaceWhitespaceBySeperator(s string) string {
-	return strings.ReplaceAll(s, " ", spmWhitespaceSep)
-}
-
-type SentencePieceModel struct {
-	maxTokenLen int
-	pre         *regexp2.Regexp
-	vocab       *Vocabulary
-}
-
-var _ TextProcessor = (*SentencePieceModel)(nil)
-
-func NewSentencePieceModel(pre string, vocab *Vocabulary) SentencePieceModel {
-	slog.Debug("Tokens", "num tokens", len(vocab.Values), "vals", vocab.Values[:5], "scores", vocab.Scores[:5], "types", vocab.Types[:5])
-
-	counter := map[int]int{}
-	var maxTokenLen int
-	for cnt := range vocab.Types {
-		switch vocab.Types[cnt] {
-		case TOKEN_TYPE_NORMAL, TOKEN_TYPE_USER_DEFINED, TOKEN_TYPE_UNUSED:
-			maxTokenLen = max(maxTokenLen, len(vocab.Values[cnt]))
-			fallthrough
-		default:
-			counter[int(vocab.Types[cnt])] += 1
-		}
-	}
-
-	slog.Debug("Token counts", "normal", counter[TOKEN_TYPE_NORMAL], "unknown", counter[TOKEN_TYPE_UNKNOWN], "control", counter[TOKEN_TYPE_CONTROL],
-		"user defined", counter[TOKEN_TYPE_USER_DEFINED], "unused", counter[TOKEN_TYPE_UNUSED], "byte", counter[TOKEN_TYPE_BYTE],
-		"max token len", maxTokenLen)
-
-	return SentencePieceModel{
-		maxTokenLen: maxTokenLen,
-		pre:         regexp2.MustCompile(pre, regexp2.Unicode|regexp2.RE2),
-		vocab:       vocab,
-	}
-}
-
-func (spm SentencePieceModel) Is(id int32, special Special) bool {
-	return spm.vocab.Is(id, special)
-}
-
-func (spm *SentencePieceModel) split(s string) iter.Seq[string] {
-	return func(yield func(string) bool) {
-		for m, _ := spm.pre.FindStringMatch(s); m != nil; m, _ = spm.pre.FindNextMatch(m) {
-			if !yield(m.String()) {
-				break
-			}
-		}
-	}
-}
-
-func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error) {
-	fragments := []fragment{{value: s}}
-	for _, special := range spm.vocab.SpecialVocabulary() {
-		// TODO: process special tokens concurrently
-		id := spm.vocab.Encode(special)
-		for i := 0; i < len(fragments); i++ {
-			frag := fragments[i]
-			if len(frag.ids) > 0 {
-				continue
-			}
-
-			var middle []fragment
-			switch i := strings.Index(frag.value, special); {
-			case i < 0:
-				middle = append(middle, frag)
-			case i > 0:
-				middle = append(middle, fragment{value: frag.value[:i]})
-				fallthrough
-			default:
-				middle = append(middle, fragment{value: special, ids: []int32{id}})
-				if rest := frag.value[i+len(special):]; rest != "" {
-					middle = append(middle, fragment{value: rest})
-				}
-			}
-
-			fragments = append(fragments[:i], append(middle, fragments[i+1:]...)...)
-		}
-	}
-	slog.Debug("fragments", "frags", fragments)
-
-	var ids []int32
-	for _, frag := range fragments {
-		if len(frag.ids) > 0 {
-			ids = append(ids, frag.ids...)
-			continue
-		}
-
-		for split := range spm.split(frag.value) {
-			split = replaceWhitespaceBySeperator(split)
-
-			var sb strings.Builder
-			sb.Write([]byte(split))
-			if id := spm.vocab.Encode(sb.String()); id >= 0 {
-				ids = append(ids, id)
-				continue
-			}
-
-			runes := []rune(sb.String())
-			pq := queue.NewWith(func(a, b any) int {
-				priA := a.(*candidate)
-				priB := b.(*candidate)
-				if priA.score > priB.score || (priA.score == priB.score && priA.a < priB.a) {
-					return -1
-				}
-				return 1
-			})
-
-			merges := make([]merge, len(runes))
-			for r := range runes {
-				merges[r] = merge{
-					p:     r - 1,
-					n:     r + 1,
-					runes: []rune{runes[r]},
-				}
-			}
-
-			slog.Debug("tokenizer", "merges", merges)
-
-			pairwise := func(a, b int) *candidate {
-				if a < 0 || b >= len(runes) {
-					return nil
-				}
-
-				left, right := string(merges[a].runes), string(merges[b].runes)
-				if id := spm.vocab.Encode(left + right); id >= 0 {
-					return &candidate{
-						a:     a,
-						b:     b,
-						score: spm.vocab.Scores[id],
-					}
-				}
-				return nil
-			}
-
-			for i := range len(runes) - 1 {
-				if pair := pairwise(i, i+1); pair != nil {
-					pq.Enqueue(pair)
-				}
-			}
-
-			pqv := pq.Values()
-			for _, v := range pqv {
-				e := v.(*candidate)
-				slog.Debug("candidate", "candidate", e)
-			}
-
-			for !pq.Empty() {
-				v, _ := pq.Dequeue()
-				pair := v.(*candidate)
-				left, right := merges[pair.a], merges[pair.b]
-
-				slog.Debug("pair", "left", left, "right", right)
-				if len(left.runes) == 0 || len(right.runes) == 0 {
-					continue
-				}
-
-				if id := spm.vocab.Encode(string(left.runes) + string(right.runes)); id < 0 {
-					continue
-				}
-
-				merges[pair.a].runes = append(left.runes, right.runes...)
-				merges[pair.b].runes = nil
-				merges[pair.a].n = right.n
-				if right.n < len(merges) {
-					merges[right.n].p = pair.a
-				}
-
-				if pair := pairwise(merges[pair.a].p, pair.a); pair != nil {
-					pq.Enqueue(pair)
-				}
-
-				if pair := pairwise(pair.a, merges[pair.a].n); pair != nil {
-					pq.Enqueue(pair)
-				}
-			}
-
-			slog.Debug("merges", "merges", merges)
-
-			for _, merge := range merges {
-				if len(merge.runes) > 0 {
-					if id := spm.vocab.Encode(string(merge.runes)); id >= 0 {
-						ids = append(ids, id)
-					} else {
-						slog.Debug("missing token", "token", string(merge.runes))
-					}
-				}
-			}
-		}
-	}
-
-	if addSpecial && len(ids) > 0 {
-		if spm.vocab.AddBOS {
-			if ids[0] == spm.vocab.BOS {
-				slog.Warn("adding bos token to prompt which already has it", "id", spm.vocab.BOS)
-			}
-
-			slog.Debug("adding bos token to prompt", "id", spm.vocab.BOS)
-			ids = append([]int32{spm.vocab.BOS}, ids...)
-		}
-
-		if spm.vocab.AddEOS {
-			if ids[len(ids)-1] == spm.vocab.EOS {
-				slog.Warn("adding eos token to prompt which already has it", "id", spm.vocab.EOS)
-			}
-
-			slog.Debug("adding eos token to prompt", "id", spm.vocab.EOS)
-			ids = append(ids, spm.vocab.EOS)
-		}
-	}
-
-	return ids, nil
-}
-
-type candidate struct {
-	a, b  int
-	score float32
-}
-
-func (spm SentencePieceModel) Decode(ids []int32) (string, error) {
-	var sb strings.Builder
-	for _, id := range ids {
-		data := spm.vocab.Decode(id)
-		data = strings.ReplaceAll(data, spmWhitespaceSep, " ")
-		if _, err := sb.WriteString(data); err != nil {
-			return "", err
-		}
-	}
-
-	slog.Debug("decoded", "ids", ids, "text", sb.String())
-	return sb.String(), nil
-}

model/process_text_spm_test.go

@@ -1,118 +0,0 @@
-package model
-
-import (
-	"log/slog"
-	"os"
-	"path/filepath"
-	"slices"
-	"testing"
-
-	"google.golang.org/protobuf/proto"
-
-	"github.com/ollama/ollama/convert/sentencepiece"
-)
-
-func loadSentencePieceVocab(t *testing.T) SentencePieceModel {
-	t.Helper()
-
-	bts, err := os.ReadFile(filepath.Join("testdata", "gemma2", "tokenizer.model"))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var spm sentencepiece.ModelProto
-	if err := proto.Unmarshal(bts, &spm); err != nil {
-		t.Fatal(err)
-	}
-
-	preTokenizer := `(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+`
-
-	var v Vocabulary
-
-	for _, piece := range spm.GetPieces() {
-		v.Values = append(v.Values, piece.GetPiece())
-		v.Scores = append(v.Scores, piece.GetScore())
-		switch t := piece.GetType(); t {
-		case sentencepiece.ModelProto_SentencePiece_UNKNOWN,
-			sentencepiece.ModelProto_SentencePiece_CONTROL,
-			sentencepiece.ModelProto_SentencePiece_UNUSED,
-			sentencepiece.ModelProto_SentencePiece_BYTE:
-			v.Types = append(v.Types, uint32(t))
-		default:
-			tt := uint32(sentencepiece.ModelProto_SentencePiece_NORMAL)
-			// todo parse the special tokens file
-			//   - this will roundtrip correctly but the <start_of_turn> and
-			//     <end_of_turn> tokens aren't processed
-			v.Types = append(v.Types, tt)
-		}
-	}
-
-	return NewSentencePieceModel(preTokenizer, &v)
-}
-
-func TestSentencePieceEncode(t *testing.T) {
-	logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}))
-	slog.SetDefault(logger)
-
-	tokenizer := loadSentencePieceVocab(t)
-
-	t.Run("basic roundtrip", func(t *testing.T) {
-		t.Parallel()
-
-		cases := []string{
-			"hello",
-			"hello ",
-			"hello  ",
-			" hello",
-			" hello ",
-			" hello  ",
-			"hello world",
-			"请考试我的软件！12345",
-			"你好",
-			"Hello 你好 world!",
-			"Special characters: !@#$%^&*()_+-=[]{}|;':\",./<>?",
-			"Multilingual: 你好 こんにちは Привет Hola مرحبا",
-			"Numbers and symbols: 123456789 +- */",
-			"Special tokens: <bos> text <eos>",
-			"Code snippets: func main() { fmt.Println(\"Hello World\") }",
-			"Long text: " + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " +
-				"Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. " +
-				"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.",
-		}
-
-		for _, want := range cases {
-			ids, err := tokenizer.Encode(want, true)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if got, err := tokenizer.Decode(ids); err != nil {
-				t.Fatal(err)
-			} else if got != want {
-				t.Errorf("got %q, want %q [%#v]", got, want, ids)
-			}
-		}
-	})
-
-	t.Run("special tokens", func(t *testing.T) {
-		type candidate struct {
-			token string
-			ids   []int32
-		}
-
-		cases := []candidate{
-			{"<bos>", []int32{2}},
-			{"<eos>", []int32{1}},
-		}
-
-		for _, want := range cases {
-			ids, err := tokenizer.Encode(want.token, true)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if !slices.Equal(ids, want.ids) {
-				t.Errorf("got %#v, want %#v", ids, want.ids)
-			}
-		}
-	})
-}

readline/readline.go

@@ -116,9 +116,19 @@ func (i *Instance) Readline() (string, error) {
 
 			switch r {
 			case KeyUp:
-				i.historyPrev(buf, &currentLineBuf)
+				if i.History.Pos > 0 {
+					if i.History.Pos == i.History.Size() {
+						currentLineBuf = []rune(buf.String())
+					}
+					buf.Replace([]rune(i.History.Prev()))
+				}
 			case KeyDown:
-				i.historyNext(buf, &currentLineBuf)
+				if i.History.Pos < i.History.Size() {
+					buf.Replace([]rune(i.History.Next()))
+					if i.History.Pos == i.History.Size() {
+						buf.Replace(currentLineBuf)
+					}
+				}
 			case KeyLeft:
 				buf.MoveLeft()
 			case KeyRight:
@@ -175,10 +185,6 @@ func (i *Instance) Readline() (string, error) {
 			esc = true
 		case CharInterrupt:
 			return "", ErrInterrupt
-		case CharPrev:
-			i.historyPrev(buf, &currentLineBuf)
-		case CharNext:
-			i.historyNext(buf, &currentLineBuf)
 		case CharLineStart:
 			buf.MoveToStart()
 		case CharLineEnd:
@@ -240,24 +246,6 @@ func (i *Instance) HistoryDisable() {
 	i.History.Enabled = false
 }
 
-func (i *Instance) historyPrev(buf *Buffer, currentLineBuf *[]rune) {
-	if i.History.Pos > 0 {
-		if i.History.Pos == i.History.Size() {
-			*currentLineBuf = []rune(buf.String())
-		}
-		buf.Replace([]rune(i.History.Prev()))
-	}
-}
-
-func (i *Instance) historyNext(buf *Buffer, currentLineBuf *[]rune) {
-	if i.History.Pos < i.History.Size() {
-		buf.Replace([]rune(i.History.Next()))
-		if i.History.Pos == i.History.Size() {
-			buf.Replace(*currentLineBuf)
-		}
-	}
-}
-
 func NewTerminal() (*Terminal, error) {
 	fd := os.Stdin.Fd()
 	termios, err := SetRawMode(fd)

runner/ollamarunner/runner.go

@@ -691,6 +691,65 @@ type EmbeddingResponse struct {
 	Embedding []float32 `json:"embedding"`
 }
 
+func (s *Server) embeddings(w http.ResponseWriter, r *http.Request) {
+	var req EmbeddingRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, fmt.Sprintf("bad request: %s", err), http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	slog.Debug("embedding request", "content", req.Content)
+
+	seq, err := s.NewSequence(req.Content, nil, NewSequenceParams{embedding: true})
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Failed to create new sequence: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	// Ensure there is a place to put the sequence, released when removed from s.seqs
+	if err := s.seqsSem.Acquire(r.Context(), 1); err != nil {
+		if errors.Is(err, context.Canceled) {
+			slog.Info("aborting embeddings request due to client closing the connection")
+		} else {
+			slog.Error("Failed to acquire semaphore", "error", err)
+		}
+		return
+	}
+
+	s.mu.Lock()
+	found := false
+	for i, sq := range s.seqs {
+		if sq == nil {
+			seq.cache, seq.inputs, err = s.cache.LoadCacheSlot(seq.inputs, req.CachePrompt)
+			if err != nil {
+				s.mu.Unlock()
+				http.Error(w, fmt.Sprintf("Failed to load cache: %v", err), http.StatusInternalServerError)
+				return
+			}
+			s.seqs[i] = seq
+			s.cond.Signal()
+			found = true
+			break
+		}
+	}
+	s.mu.Unlock()
+
+	if !found {
+		http.Error(w, "could not find an available sequence", http.StatusInternalServerError)
+		return
+	}
+
+	embedding := <-seq.embedding
+
+	if err := json.NewEncoder(w).Encode(&EmbeddingResponse{
+		Embedding: embedding,
+	}); err != nil {
+		http.Error(w, fmt.Sprintf("failed to encode response: %v", err), http.StatusInternalServerError)
+	}
+}
+
 type HealthResponse struct {
 	Status   string  `json:"status"`
 	Progress float32 `json:"progress"`
@@ -868,13 +927,9 @@ func Execute(args []string) error {
 	defer listener.Close()
 
 	mux := http.NewServeMux()
-	// TODO: support embeddings
-	mux.HandleFunc("POST /embedding", func(w http.ResponseWriter, r *http.Request) {
-		http.Error(w, "this model does not support embeddings", http.StatusNotImplemented)
-	})
-
-	mux.HandleFunc("POST /completion", server.completion)
-	mux.HandleFunc("GET /health", server.health)
+	mux.HandleFunc("/embedding", server.embeddings)
+	mux.HandleFunc("/completion", server.completion)
+	mux.HandleFunc("/health", server.health)
 
 	httpServer := http.Server{
 		Handler: mux,

sample/samplers.go

@@ -84,12 +84,14 @@ func (s *Sampler) sample(tokens []token) (token, error) {
 		return greedy(tokens), nil
 	}
 
-	// topK also sorts the tokens in descending order of logits
-	tokens = topK(tokens, s.topK)
+	if s.topK > 0 {
+		tokens = topK(tokens, s.topK)
+	} else {
+		sortLogits(tokens)
+	}
 
 	tokens = temperature(tokens, s.temperature)
 	tokens = softmax(tokens)
-
 	tokens = topP(tokens, s.topP)
 	tokens = minP(tokens, s.minP)
 

sample/transforms.go

@@ -1,58 +1,17 @@
 package sample
 
 import (
-	"container/heap"
 	"math"
 	"slices"
 )
 
-// tokenHeap implements heap.Interface and holds tokens as a min-heap to track k largest elements
-type tokenHeap []token
-
-func (h tokenHeap) Len() int           { return len(h) }
-func (h tokenHeap) Less(i, j int) bool { return h[i].value < h[j].value }
-func (h tokenHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
-
-func (h *tokenHeap) Push(x any) {
-	*h = append(*h, x.(token))
-}
-
-func (h *tokenHeap) Pop() any {
-	old := *h
-	n := len(old)
-	x := old[n-1]
-	*h = old[0 : n-1]
-	return x
-}
-
-// temperature applies scaling to the logits
-func temperature(ts []token, temp float32) []token {
-	// Ensure temperature clipping near 0 to avoid numerical instability
-	temp = max(temp, 1e-7)
-	for i := range ts {
-		ts[i].value = ts[i].value / temp
-	}
-	return ts
-}
-
-// softmax applies normalization to the logits
 func softmax(ts []token) []token {
-	// Find max logit for numerical stability
-	maxLogit := float32(math.Inf(-1))
-	for _, t := range ts {
-		if t.value > maxLogit {
-			maxLogit = t.value
-		}
-	}
-
-	// Compute exp(x - max)
 	var sum float32
 	for i, v := range ts {
-		ts[i].value = float32(math.Exp(float64(v.value - maxLogit)))
+		ts[i].value = float32(math.Exp(float64(v.value)))
 		sum += ts[i].value
 	}
 
-	// exp(x - max) / sum(exp(x - max))
 	for i := range ts {
 		ts[i].value /= sum
 	}
@@ -60,42 +19,83 @@ func softmax(ts []token) []token {
 	return ts
 }
 
-// topK limits the number of tokens considered to the k highest logits
-func topK(ts []token, k int) []token {
-	if k >= len(ts) || k <= 0 {
-		slices.SortFunc(ts, func(a, b token) int {
-			switch {
-			case a.value < b.value:
-				return 1
-			case a.value > b.value:
-				return -1
-			default:
-				return 0
-			}
-		})
-		return ts
+func temperature(ti []token, t float32) []token {
+	if t == 1 {
+		return ti
 	}
 
-	// Initialize min-heap with first k elements
-	h := make(tokenHeap, k)
-	copy(h, ts[:k])
-	heap.Init(&h)
-
-	// Process remaining elements
-	for i := k; i < len(ts); i++ {
-		if ts[i].value > h[0].value {
-			heap.Pop(&h)
-			heap.Push(&h, ts[i])
+	temp := max(t, 1e-7)
+	maxLogit := float32(math.Inf(-1))
+	for _, token := range ti {
+		if token.value > maxLogit {
+			maxLogit = token.value
 		}
 	}
 
-	// Convert heap to sorted slice in descending order
-	result := make([]token, len(h))
-	for i := k - 1; i >= 0; i-- {
-		result[i] = heap.Pop(&h).(token)
+	// subtracting max logit to avoid under/overflow
+	for i := range ti {
+		ti[i].value = (ti[i].value - maxLogit) / temp
 	}
 
-	return result
+	return ti
+}
+
+// siftDown maintains a min-heap property by recursively moving larger elements down the heap.
+//
+// The heap is represented as an array where for any node at index i:
+// - Left child is at index 2i + 1
+// - Right child is at index 2i + 2
+// - Parent is at index (i-1)/2
+//
+// The function compares a node with its children and:
+// 1. Finds the smallest value between the node and its children
+// 2. If the node is not the smallest, swaps it with its smallest child
+// 3. Continues this process down the affected path until the min-heap property is restored
+func siftDown(data []token, start, end int) {
+	root := start
+	for {
+		child := 2*root + 1
+		if child >= end {
+			break
+		}
+		// Find smaller child (we want min heap)
+		if child+1 < end && data[child+1].value < data[child].value {
+			child++
+		}
+		// Exit if root is already smaller than children
+		if data[root].value <= data[child].value {
+			break
+		}
+		// Swap with smaller child and continue
+		data[root], data[child] = data[child], data[root]
+		root = child
+	}
+}
+
+// topK limits the number of tokens considered to the k highest logits
+func topK(ts []token, k int) []token {
+	if k >= len(ts) {
+		return ts
+	}
+	// Heapify + siftDown - O(nlog(k))
+	// Build min-heap of first k elements
+	heap := ts[:k]
+	for i := k/2 - 1; i >= 0; i-- {
+		siftDown(heap, i, k)
+	}
+
+	// Process remaining elements - if larger than heap root, replace root
+	for i := k; i < len(ts); i++ {
+		if ts[i].value > heap[0].value {
+			heap[0] = ts[i]
+			siftDown(heap, 0, k)
+		}
+	}
+
+	slices.Reverse(heap)
+
+	ts = heap
+	return ts
 }
 
 // topP limits tokens to those with cumulative probability p
@@ -143,3 +143,61 @@ func minP(ts []token, p float32) []token {
 	ts = validTokens
 	return ts
 }
+
+// TODO(parthsareen): possibly replace with simpler implementation https://github.com/ollama/ollama/issues/9584
+// Conting sort implementation to sort tokens by logits
+func sortLogits(tokens []token) {
+	if len(tokens) <= 1 {
+		return
+	}
+
+	// Find max/min in a single pass
+	minLogit, maxLogit := tokens[0].value, tokens[0].value
+	for _, t := range tokens[1:] {
+		if t.value < minLogit {
+			minLogit = t.value
+		} else if t.value > maxLogit {
+			maxLogit = t.value
+		}
+	}
+
+	// Calculate scaling to map to uint32 range
+	logitRange := maxLogit - minLogit
+	if logitRange < 1e-6 {
+		return // All values effectively equal
+	}
+
+	// Count frequencies directly from tokens
+	const maxInt = (1 << 24) - 1 // Use 24 bits for good granularity
+	var counts [256]int          // For first byte
+
+	// First pass: count frequencies
+	for _, t := range tokens {
+		// Map to [0, maxInt] range
+		score := min(uint32((t.value-minLogit)*float32(maxInt)/logitRange), maxInt)
+		counts[score>>16]++
+	}
+
+	// Calculate offsets
+	var offset int
+	for i := range counts {
+		count := counts[i]
+		counts[i] = offset
+		offset += count
+	}
+
+	// Second pass: place elements in correct position
+	output := make([]token, len(tokens))
+	// Track current positions
+	countsCopy := counts
+
+	for i, t := range tokens {
+		score := min(uint32((t.value-minLogit)*float32(maxInt)/logitRange), maxInt)
+
+		pos := countsCopy[score>>16]
+		countsCopy[score>>16]++
+		output[len(tokens)-1-pos] = tokens[i]
+	}
+
+	copy(tokens, output)
+}

sample/transforms_test.go

@@ -6,155 +6,86 @@ import (
 	"testing"
 )
 
-// Helper to convert float32 slice to logit slice
-func toTokens(values []float32) []token {
+// Helper to convert float64 slice to logit slice
+func toTokens(values []float64) []token {
 	tokens := make([]token, len(values))
 	for i, v := range values {
 		tokens[i] = token{
 			id:    int32(i),
-			value: v,
+			value: float32(v),
 		}
 	}
 	return tokens
 }
 
 // Helper to compare logit slices
-func compareLogits(t *testing.T, name string, want []float32, got []token) {
+func compareLogits(t *testing.T, name string, want []float64, got []token) {
 	t.Helper()
 	if len(want) != len(got) {
 		t.Errorf("%s: length mismatch: want %d, got %d", name, len(want), len(got))
 		return
 	}
 	for i := range want {
-		if math.Abs(float64(got[i].value-want[i])) > 1e-6 {
+		if math.Abs(float64(got[i].value)-want[i]) > 1e-6 {
 			t.Errorf("%s: index %d: want %f, got %f", name, i, want[i], got[i].value)
 		}
 	}
 }
 
 func TestTemperature(t *testing.T) {
-	input := []float32{1.0, 4.0, -2.0, 0.0}
+	input := []float64{2, -1, 4, -3, 1, -2, 0}
+	want := []float64{-4, -10, 0, -14, -6, -12, -8} // (logit - max logit) / temp
+
 	got := temperature(toTokens(input), 0.5)
-	want := []float32{2.0, 8.0, -4.0, 0.0}
-	compareLogits(t, "temperature(0.5)", want, got)
-
-	got = temperature(toTokens(input), 1.0)
-	want = []float32{1.0, 4.0, -2.0, 0.0}
-	compareLogits(t, "temperature(1)", want, got)
-
-	got = temperature(toTokens(input), 0.0)
-	want = []float32{1e7, 4e7, -2e7, 0.0}
-	compareLogits(t, "temperature(0)", want, got)
+	compareLogits(t, "Temperature", want, got)
 }
 
 func TestSoftmax(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    []float32
-		expected []float32
-	}{
-		{
-			name:     "correctness softmax",
-			input:    []float32{1, -2, 3, 0},
-			expected: []float32{0.113550, 0.005653, 0.839024, 0.041773},
-		},
-		{
-			name:  "normal distribution",
-			input: []float32{0.026986899, 0.043722924, 0.036774673, 0.27755088, 0.0046718004, 0.08582123, 0.20409796, 0.00412893, 0.15720603, 0.045046154, 0.0030491839, 0.01681367},
-		},
-		{
-			name:  "single value",
-			input: []float32{1.0},
-		},
-		{
-			name:  "identical values",
-			input: []float32{0.9, 0.9, 0.9},
-		},
-		{
-			name:  "large values",
-			input: []float32{1000.0, 2000.0, 3000.0},
-		},
-		{
-			name:  "small values",
-			input: []float32{1e-6, 2e-6, 3e-6},
-		},
-		{
-			name:  "negative values",
-			input: []float32{-1.0, -2.0, -3.0},
-		},
-		{
-			name:  "mixed values",
-			input: []float32{-100.0, 0.0, 100.0},
-		},
+	input := []float64{-3, -2, -1, 0, 1, 2, 4}
+	got := softmax(toTokens(input))
+
+	// Check probabilities sum to 1
+	var sum float32
+	for _, token := range got {
+		sum += token.value
+	}
+	if math.Abs(float64(sum)-1.0) > 1e-6 {
+		t.Errorf("probabilities don't sum to 1: got %f", sum)
 	}
 
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := softmax(toTokens(tt.input))
-
-			if tt.expected != nil {
-				compareLogits(t, tt.name, tt.expected, got)
-				return
-			}
-
-			// Check probabilities sum to 1
-			var sum float32
-			for _, token := range got {
-				sum += token.value
-				if token.value < 0 || token.value > 1 {
-					t.Errorf("probability out of range [0,1]: got %f", token.value)
-				}
-			}
-			if math.Abs(float64(sum-1.0)) > 1e-6 {
-				t.Errorf("probabilities don't sum to 1: got %f", sum)
-			}
-		})
+	// Check relative ordering is preserved
+	for i := 1; i < len(got); i++ {
+		if got[i].value < got[i-1].value {
+			t.Errorf("probability ordering not preserved at index %d", i)
+		}
 	}
 }
 
 func TestTopK(t *testing.T) {
-	input := []float32{0.026986899, 0.043722924, 0.036774673, 0.27755088, 0.0046718004, 0.08582123, 0.20409796, 0.00412893, 0.15720603, 0.045046154, 0.0030491839, 0.01681367}
+	input := []float64{-3, -2, -1, 0, 1, 2, 4}
 
-	// Test k=5
-	got := topK(toTokens(input), 5)
-	if len(got) != 5 {
-		t.Errorf("topK(5): wrong length: want 5, got %d", len(got))
+	// Test k=3
+	got := topK(toTokens(input), 3)
+	if len(got) != 3 {
+		t.Errorf("topK(3): wrong length: want 3, got %d", len(got))
 	}
-	// Should keep highest 3 values in descending order
-	want := []float32{0.27755088, 0.20409796, 0.15720603, 0.08582123, 0.045046154}
+	// Should keep highest 3 values: 4, 2, 1
+	want := []float64{4, 2, 1}
 	compareLogits(t, "topK(3)", want, got)
 
-	got = topK(toTokens(input), 20)
-	if len(got) != len(input) {
-		t.Errorf("topK(20): wrong length: want %d, got %d", len(input), len(got))
-	}
-
-	// Test k=-1
-	input = []float32{0.026986899, 0.043722924, 0.036774673, 0.27755088, 0.0046718004, 0.08582123, 0.20409796, 0.00412893, 0.15720603, 0.045046154, 0.0030491839, 0.01681367}
-	want = []float32{0.27755088, 0.20409796, 0.15720603, 0.08582123, 0.045046154, 0.043722924, 0.036774673, 0.026986899, 0.01681367, 0.0046718004, 0.00412893, 0.0030491839}
-	got = topK(toTokens(input), -1)
-	if len(got) != len(input) {
-		t.Errorf("topK(-1): wrong length: want %d, got %d", len(input), len(got))
-	}
-	compareLogits(t, "topK(-1)", want, got)
-
-	// Test k=0
-	input = []float32{0.026986899, 0.043722924, 0.036774673, 0.27755088, 0.0046718004, 0.08582123, 0.20409796, 0.00412893, 0.15720603, 0.045046154, 0.0030491839, 0.01681367}
-	want = []float32{0.27755088, 0.20409796, 0.15720603, 0.08582123, 0.045046154, 0.043722924, 0.036774673, 0.026986899, 0.01681367, 0.0046718004, 0.00412893, 0.0030491839}
-	got = topK(toTokens(input), 0)
-	if len(got) != len(input) {
-		t.Errorf("topK(-1): wrong length: want %d, got %d", len(input), len(got))
-	}
-	compareLogits(t, "topK(-1)", want, got)
+	// Test k > len
+	got = topK(toTokens(input), 10)
+	compareLogits(t, "topK(10)", input, got)
 }
 
 func TestTopP(t *testing.T) {
-	input := []float32{-3, -2, -1, 0, 1, 2, 4}
+	input := []float64{-3, -2, -1, 0, 1, 2, 4}
 	tokens := toTokens(input)
 
 	// First apply temperature and softmax to get probabilities
+	tokens = temperature(tokens, 1)
 	tokens = softmax(tokens)
-	tokens = topK(tokens, 20)
+	sortLogits(tokens)
 
 	// Then apply topP
 	got := topP(tokens, 0.95)
@@ -167,10 +98,11 @@ func TestTopP(t *testing.T) {
 }
 
 func TestMinP(t *testing.T) {
-	input := []float32{-3, -2, -1, 0, 1, 2, 4, 3}
+	input := []float64{-3, -2, -1, 0, 1, 2, 4, 3}
 	tokens := toTokens(input)
 
 	// First apply temperature and softmax
+	tokens = temperature(tokens, 1)
 	tokens = softmax(tokens)
 
 	// Then apply minP
@@ -183,10 +115,10 @@ func TestMinP(t *testing.T) {
 }
 
 func TestSortLogits(t *testing.T) {
-	input := []float32{0.026986899, 0.043722924, 0.036774673, 0.27755088, 0.0046718004, 0.08582123, 0.20409796, 0.00412893, 0.15720603, 0.045046154, 0.0030491839, 0.01681367}
+	input := []float64{3, 1, 4, 2, -1, 0, -2}
 	tokens := toTokens(input)
 
-	tokens = topK(tokens, 20)
+	sortLogits(tokens)
 
 	for i := 1; i < len(tokens); i++ {
 		if tokens[i].value > tokens[i-1].value {
@@ -195,7 +127,7 @@ func TestSortLogits(t *testing.T) {
 		}
 	}
 
-	want := []float32{0.27755088, 0.20409796, 0.15720603, 0.08582123, 0.045046154, 0.043722924, 0.036774673, 0.026986899, 0.01681367, 0.0046718004, 0.00412893, 0.0030491839}
+	want := []float64{4, 3, 2, 1, 0, -1, -2}
 	compareLogits(t, "sortLogits", want, tokens)
 }
 
@@ -219,14 +151,6 @@ func BenchmarkTransforms(b *testing.B) {
 		}
 	})
 
-	b.Run("Softmax", func(b *testing.B) {
-		b.ResetTimer()
-		for b.Loop() {
-			copy(tokensCopy, tokens)
-			softmax(tokensCopy)
-		}
-	})
-
 	b.Run("TopK", func(b *testing.B) {
 		b.ResetTimer()
 		for b.Loop() {
@@ -255,7 +179,7 @@ func BenchmarkTransforms(b *testing.B) {
 		b.ResetTimer()
 		for b.Loop() {
 			copy(tokensCopy, tokens)
-			topK(tokensCopy, 200000)
+			sortLogits(tokensCopy)
 		}
 	})
 }

scripts/build_windows.ps1

@@ -80,14 +80,13 @@ function checkEnv() {
 
 
 function buildOllama() {
-    mkdir -Force -path "${script:DIST_DIR}\"
     if ($script:ARCH -ne "arm64") {
         Remove-Item -ea 0 -recurse -force -path "${script:SRC_DIR}\dist\windows-${script:ARCH}"
         New-Item "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\" -ItemType Directory -ea 0
 
         & cmake --fresh --preset CPU --install-prefix $script:DIST_DIR
         if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-        & cmake --build --preset CPU  --config Release --parallel $script:JOBS
+        & cmake --build --preset CPU --parallel $script:JOBS
         if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
         & cmake --install build --component CPU --strip
         if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
@@ -102,7 +101,7 @@ function buildOllama() {
             # to avoid 2022 (or newer) from being used as the default
             & cmake --fresh --preset "CUDA 11" -G "Visual Studio 16 2019" --install-prefix $script:DIST_DIR
             if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-            & cmake --build --preset "CUDA 11"  --config Release --parallel $script:JOBS
+            & cmake --build --preset "CUDA 11" --parallel $script:JOBS
             if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
             & cmake --install build --component "CUDA" --strip
             if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
@@ -113,7 +112,7 @@ function buildOllama() {
             write-host "Building CUDA v12 backend libraries"
             & cmake --fresh --preset "CUDA 12" --install-prefix $script:DIST_DIR
             if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-            & cmake --build --preset "CUDA 12"  --config Release --parallel $script:JOBS
+            & cmake --build --preset "CUDA 12" --parallel $script:JOBS
             if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
             & cmake --install build --component "CUDA" --strip
             if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
@@ -132,7 +131,7 @@ function buildOllama() {
             $env:HIPCXX=""
             $env:HIP_PLATFORM=""
             $env:CMAKE_PREFIX_PATH=""
-            & cmake --build --preset "ROCm"  --config Release --parallel $script:JOBS
+            & cmake --build --preset "ROCm" --parallel $script:JOBS
             if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
             & cmake --install build --component "HIP" --strip
             if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}

server/internal/cache/blob/cache.go

@@ -146,7 +146,7 @@ func debugger(err *error) func(step string) {
 // be in either of the following forms:
 //
 //	@<digest>
-//	<name>@<digest>
+//	<name>
 //	<name>
 //
 // If a digest is provided, it is returned as is and nothing else happens.
@@ -160,6 +160,8 @@ func debugger(err *error) func(step string) {
 // hashed is passed to a PutBytes call to ensure that the manifest is in the
 // blob store. This is done to ensure that future calls to [Get] succeed in
 // these cases.
+//
+// TODO(bmizerany): Move Links/Resolve/etc. out of this package.
 func (c *DiskCache) Resolve(name string) (Digest, error) {
 	name, digest := splitNameDigest(name)
 	if digest != "" {
@@ -277,6 +279,18 @@ func (c *DiskCache) Get(d Digest) (Entry, error) {
 // It returns an error if either the name or digest is invalid, or if link
 // creation encounters any issues.
 func (c *DiskCache) Link(name string, d Digest) error {
+	// TODO(bmizerany): Move link handling from cache to registry.
+	//
+	// We originally placed links in the cache due to its storage
+	// knowledge. However, the registry likely offers better context for
+	// naming concerns, and our API design shouldn't be tightly coupled to
+	// our on-disk format.
+	//
+	// Links work effectively when independent from physical location -
+	// they can reference content with matching SHA regardless of storage
+	// location. In an upcoming change, we plan to shift this
+	// responsibility to the registry where it better aligns with the
+	// system's conceptual model.
 	manifest, err := c.manifestPath(name)
 	if err != nil {
 		return err
@@ -327,9 +341,7 @@ func (c *DiskCache) GetFile(d Digest) string {
 	return absJoin(c.dir, "blobs", filename)
 }
 
-// Links returns a sequence of link names. The sequence is in lexical order.
-// Names are converted from their relative path form to their name form but are
-// not guaranteed to be valid. Callers should validate the names before using.
+// Links returns a sequence of links in the cache in lexical order.
 func (c *DiskCache) Links() iter.Seq2[string, error] {
 	return func(yield func(string, error) bool) {
 		for path, err := range c.links() {
@@ -402,14 +414,12 @@ func (c *DiskCache) links() iter.Seq2[string, error] {
 }
 
 type checkWriter struct {
-	size int64
 	d    Digest
-	f    *os.File
+	size int64
+	n    int64
 	h    hash.Hash
-
-	w   io.Writer // underlying writer; set by creator
-	n   int64
-	err error
+	f    *os.File
+	err  error
 
 	testHookBeforeFinalWrite func(*os.File)
 }
@@ -425,10 +435,6 @@ func (w *checkWriter) seterr(err error) error {
 // underlying writer is guaranteed to be the last byte of p as verified by the
 // hash.
 func (w *checkWriter) Write(p []byte) (int, error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-
 	_, err := w.h.Write(p)
 	if err != nil {
 		return 0, w.seterr(err)
@@ -447,7 +453,7 @@ func (w *checkWriter) Write(p []byte) (int, error) {
 	if nextSize > w.size {
 		return 0, w.seterr(fmt.Errorf("content exceeds expected size: %d > %d", nextSize, w.size))
 	}
-	n, err := w.w.Write(p)
+	n, err := w.f.Write(p)
 	w.n += int64(n)
 	return n, w.seterr(err)
 }
@@ -487,12 +493,10 @@ func (c *DiskCache) copyNamedFile(name string, file io.Reader, out Digest, size
 
 	// Copy file to f, but also into h to double-check hash.
 	cw := &checkWriter{
-		d:    out,
-		size: size,
-		h:    sha256.New(),
-		f:    f,
-		w:    f,
-
+		d:                        out,
+		size:                     size,
+		h:                        sha256.New(),
+		f:                        f,
 		testHookBeforeFinalWrite: c.testHookBeforeFinalWrite,
 	}
 	n, err := io.Copy(cw, file)
@@ -528,6 +532,11 @@ func splitNameDigest(s string) (name, digest string) {
 var errInvalidName = errors.New("invalid name")
 
 func nameToPath(name string) (_ string, err error) {
+	if strings.Contains(name, "@") {
+		// TODO(bmizerany): HACK: Fix names.Parse to validate.
+		// TODO(bmizerany): merge with default parts (maybe names.Merge(a, b))
+		return "", errInvalidName
+	}
 	n := names.Parse(name)
 	if !n.IsFullyQualified() {
 		return "", errInvalidName
@@ -538,7 +547,8 @@ func nameToPath(name string) (_ string, err error) {
 func absJoin(pp ...string) string {
 	abs, err := filepath.Abs(filepath.Join(pp...))
 	if err != nil {
-		panic(err) // this should never happen
+		// Likely a bug bug or a bad OS problem. Just panic.
+		panic(err)
 	}
 	return abs
 }

server/internal/cache/blob/chunked.go

@@ -1,73 +0,0 @@
-package blob
-
-import (
-	"crypto/sha256"
-	"errors"
-	"io"
-	"os"
-)
-
-// Chunk represents a range of bytes in a blob.
-type Chunk struct {
-	Start int64
-	End   int64
-}
-
-// Size returns end minus start plus one.
-func (c Chunk) Size() int64 {
-	return c.End - c.Start + 1
-}
-
-// Chunker writes to a blob in chunks.
-// Its zero value is invalid. Use [DiskCache.Chunked] to create a new Chunker.
-type Chunker struct {
-	digest Digest
-	size   int64
-	f      *os.File // nil means pre-validated
-}
-
-// Chunked returns a new Chunker, ready for use storing a blob of the given
-// size in chunks.
-//
-// Use [Chunker.Put] to write data to the blob at specific offsets.
-func (c *DiskCache) Chunked(d Digest, size int64) (*Chunker, error) {
-	name := c.GetFile(d)
-	info, err := os.Stat(name)
-	if err == nil && info.Size() == size {
-		return &Chunker{}, nil
-	}
-	f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0o666)
-	if err != nil {
-		return nil, err
-	}
-	return &Chunker{digest: d, size: size, f: f}, nil
-}
-
-// Put copies chunk.Size() bytes from r to the blob at the given offset,
-// merging the data with the existing blob. It returns an error if any. As a
-// special case, if r has less than chunk.Size() bytes, Put returns
-// io.ErrUnexpectedEOF.
-func (c *Chunker) Put(chunk Chunk, d Digest, r io.Reader) error {
-	if c.f == nil {
-		return nil
-	}
-
-	cw := &checkWriter{
-		d:    d,
-		size: chunk.Size(),
-		h:    sha256.New(),
-		f:    c.f,
-		w:    io.NewOffsetWriter(c.f, chunk.Start),
-	}
-
-	_, err := io.CopyN(cw, r, chunk.Size())
-	if err != nil && errors.Is(err, io.EOF) {
-		return io.ErrUnexpectedEOF
-	}
-	return err
-}
-
-// Close closes the underlying file.
-func (c *Chunker) Close() error {
-	return c.f.Close()
-}

server/internal/cache/blob/digest.go

@@ -63,10 +63,6 @@ func (d Digest) Short() string {
 	return fmt.Sprintf("%x", d.sum[:4])
 }
 
-func (d Digest) Sum() [32]byte {
-	return d.sum
-}
-
 func (d Digest) Compare(other Digest) int {
 	return slices.Compare(d.sum[:], other.sum[:])
 }

server/internal/chunks/chunks.go

@@ -0,0 +1,78 @@
+package chunks
+
+import (
+	"fmt"
+	"iter"
+	"strconv"
+	"strings"
+)
+
+type Chunk struct {
+	Start, End int64
+}
+
+func New(start, end int64) Chunk {
+	return Chunk{start, end}
+}
+
+// ParseRange parses a string in the form "unit=range" where unit is a string
+// and range is a string in the form "start-end". It returns the unit and the
+// range as a Chunk.
+func ParseRange(s string) (unit string, _ Chunk, _ error) {
+	unit, r, _ := strings.Cut(s, "=")
+	if r == "" {
+		return unit, Chunk{}, nil
+	}
+	c, err := Parse(r)
+	if err != nil {
+		return "", Chunk{}, err
+	}
+	return unit, c, err
+}
+
+// Parse parses a string in the form "start-end" and returns the Chunk.
+func Parse(s string) (Chunk, error) {
+	startStr, endStr, _ := strings.Cut(s, "-")
+	start, err := strconv.ParseInt(startStr, 10, 64)
+	if err != nil {
+		return Chunk{}, fmt.Errorf("invalid start: %v", err)
+	}
+	end, err := strconv.ParseInt(endStr, 10, 64)
+	if err != nil {
+		return Chunk{}, fmt.Errorf("invalid end: %v", err)
+	}
+	if start > end {
+		return Chunk{}, fmt.Errorf("invalid range %d-%d: start > end", start, end)
+	}
+	return Chunk{start, end}, nil
+}
+
+// Of returns a sequence of contiguous Chunks of size chunkSize that cover
+// the range [0, size), in order.
+func Of(size, chunkSize int64) iter.Seq[Chunk] {
+	return func(yield func(Chunk) bool) {
+		for start := int64(0); start < size; start += chunkSize {
+			end := min(start+chunkSize-1, size-1)
+			if !yield(Chunk{start, end}) {
+				break
+			}
+		}
+	}
+}
+
+// Count returns the number of Chunks of size chunkSize needed to cover the
+// range [0, size).
+func Count(size, chunkSize int64) int64 {
+	return (size + chunkSize - 1) / chunkSize
+}
+
+// Size returns end minus start plus one.
+func (c Chunk) Size() int64 {
+	return c.End - c.Start + 1
+}
+
+// String returns the string representation of the Chunk in the form
+// "{start}-{end}".
+func (c Chunk) String() string {
+	return fmt.Sprintf("%d-%d", c.Start, c.End)
+}

server/internal/chunks/chunks_test.go

@@ -0,0 +1,65 @@
+package chunks
+
+import (
+	"slices"
+	"testing"
+)
+
+func TestOf(t *testing.T) {
+	cases := []struct {
+		total     int64
+		chunkSize int64
+		want      []Chunk
+	}{
+		{0, 1, nil},
+		{1, 1, []Chunk{{0, 0}}},
+		{1, 2, []Chunk{{0, 0}}},
+		{2, 1, []Chunk{{0, 0}, {1, 1}}},
+		{10, 9, []Chunk{{0, 8}, {9, 9}}},
+	}
+
+	for _, tt := range cases {
+		got := slices.Collect(Of(tt.total, tt.chunkSize))
+		if !slices.Equal(got, tt.want) {
+			t.Errorf("[%d/%d]: got %v; want %v", tt.total, tt.chunkSize, got, tt.want)
+		}
+	}
+}
+
+func TestSize(t *testing.T) {
+	cases := []struct {
+		c    Chunk
+		want int64
+	}{
+		{Chunk{0, 0}, 1},
+		{Chunk{0, 1}, 2},
+		{Chunk{3, 4}, 2},
+	}
+
+	for _, tt := range cases {
+		got := tt.c.Size()
+		if got != tt.want {
+			t.Errorf("%v: got %d; want %d", tt.c, got, tt.want)
+		}
+	}
+}
+
+func TestCount(t *testing.T) {
+	cases := []struct {
+		total     int64
+		chunkSize int64
+		want      int64
+	}{
+		{0, 1, 0},
+		{1, 1, 1},
+		{1, 2, 1},
+		{2, 1, 2},
+		{10, 9, 2},
+	}
+	for _, tt := range cases {
+		got := Count(tt.total, tt.chunkSize)
+		if got != tt.want {
+			t.Errorf("[%d/%d]: got %d; want %d", tt.total, tt.chunkSize, got, tt.want)
+		}
+	}
+}

server/internal/client/ollama/registry.go

@@ -19,7 +19,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"iter"
 	"log/slog"
 	"net/http"
 	"os"
@@ -36,8 +35,10 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/ollama/ollama/server/internal/cache/blob"
+	"github.com/ollama/ollama/server/internal/chunks"
 	"github.com/ollama/ollama/server/internal/internal/backoff"
 	"github.com/ollama/ollama/server/internal/internal/names"
+	"github.com/ollama/ollama/server/internal/internal/syncs"
 
 	_ "embed"
 )
@@ -65,7 +66,12 @@ var (
 const (
 	// DefaultChunkingThreshold is the threshold at which a layer should be
 	// split up into chunks when downloading.
-	DefaultChunkingThreshold = 64 << 20
+	DefaultChunkingThreshold = 128 << 20
+
+	// DefaultMaxChunkSize is the default maximum size of a chunk to
+	// download. It is configured based on benchmarks and aims to strike a
+	// balance between download speed and memory usage.
+	DefaultMaxChunkSize = 8 << 20
 )
 
 var defaultCache = sync.OnceValues(func() (*blob.DiskCache, error) {
@@ -205,7 +211,8 @@ type Registry struct {
 	// pushing or pulling models. If zero, the number of streams is
 	// determined by [runtime.GOMAXPROCS].
 	//
-	// A negative value means no limit.
+	// Clients that want "unlimited" streams should set this to a large
+	// number.
 	MaxStreams int
 
 	// ChunkingThreshold is the maximum size of a layer to download in a single
@@ -275,13 +282,24 @@ func DefaultRegistry() (*Registry, error) {
 }
 
 func (r *Registry) maxStreams() int {
-	return cmp.Or(r.MaxStreams, runtime.GOMAXPROCS(0))
+	n := cmp.Or(r.MaxStreams, runtime.GOMAXPROCS(0))
+
+	// Large downloads require a writter stream, so ensure we have at least
+	// two streams to avoid a deadlock.
+	return max(n, 2)
 }
 
 func (r *Registry) maxChunkingThreshold() int64 {
 	return cmp.Or(r.ChunkingThreshold, DefaultChunkingThreshold)
 }
 
+// chunkSizeFor returns the chunk size for a layer of the given size. If the
+// size is less than or equal to the max chunking threshold, the size is
+// returned; otherwise, the max chunk size is returned.
+func (r *Registry) maxChunkSize() int64 {
+	return cmp.Or(r.MaxChunkSize, DefaultMaxChunkSize)
+}
+
 type PushParams struct {
 	// From is an optional destination name for the model. If empty, the
 	// destination name is the same as the source name.
@@ -408,21 +426,6 @@ func canRetry(err error) bool {
 	return re.Status >= 500
 }
 
-// trackingReader is an io.Reader that tracks the number of bytes read and
-// calls the update function with the layer, the number of bytes read.
-//
-// It always calls update with a nil error.
-type trackingReader struct {
-	r io.Reader
-	n *atomic.Int64
-}
-
-func (r *trackingReader) Read(p []byte) (n int, err error) {
-	n, err = r.r.Read(p)
-	r.n.Add(int64(n))
-	return
-}
-
 // Pull pulls the model with the given name from the remote registry into the
 // cache.
 //
@@ -431,6 +434,11 @@ func (r *trackingReader) Read(p []byte) (n int, err error) {
 // typically slower than splitting the model up across layers, and is mostly
 // utilized for layers of type equal to "application/vnd.ollama.image".
 func (r *Registry) Pull(ctx context.Context, name string) error {
+	scheme, n, _, err := r.parseNameExtended(name)
+	if err != nil {
+		return err
+	}
+
 	m, err := r.Resolve(ctx, name)
 	if err != nil {
 		return err
@@ -449,95 +457,126 @@ func (r *Registry) Pull(ctx context.Context, name string) error {
 		return err == nil && info.Size == l.Size
 	}
 
+	t := traceFromContext(ctx)
+
+	g, ctx := errgroup.WithContext(ctx)
+	g.SetLimit(r.maxStreams())
+
 	layers := m.Layers
 	if m.Config != nil && m.Config.Digest.IsValid() {
 		layers = append(layers, m.Config)
 	}
 
-	// Send initial layer trace events to allow clients to have an
-	// understanding of work to be done before work starts.
-	t := traceFromContext(ctx)
-	skip := make([]bool, len(layers))
-	for i, l := range layers {
-		t.update(l, 0, nil)
+	for _, l := range layers {
 		if exists(l) {
-			skip[i] = true
 			t.update(l, l.Size, ErrCached)
-		}
-	}
-
-	g, ctx := errgroup.WithContext(ctx)
-	g.SetLimit(r.maxStreams())
-	for i, l := range layers {
-		if skip[i] {
 			continue
 		}
 
-		chunked, err := c.Chunked(l.Digest, l.Size)
+		blobURL := fmt.Sprintf("%s://%s/v2/%s/%s/blobs/%s", scheme, n.Host(), n.Namespace(), n.Model(), l.Digest)
+		req, err := r.newRequest(ctx, "GET", blobURL, nil)
 		if err != nil {
 			t.update(l, 0, err)
 			continue
 		}
-		defer chunked.Close()
 
-		var progress atomic.Int64
-		for cs, err := range r.chunksums(ctx, name, l) {
-			if err != nil {
-				t.update(l, progress.Load(), err)
-				break
-			}
+		t.update(l, 0, nil)
+
+		if l.Size <= r.maxChunkingThreshold() {
+			g.Go(func() error {
+				// TODO(bmizerany): retry/backoff like below in
+				// the chunking case
+				res, err := sendRequest(r.client(), req)
+				if err != nil {
+					return err
+				}
+				defer res.Body.Close()
+				err = c.Put(l.Digest, res.Body, l.Size)
+				if err == nil {
+					t.update(l, l.Size, nil)
+				}
+				return err
+			})
+		} else {
+			q := syncs.NewRelayReader()
 
 			g.Go(func() (err error) {
-				defer func() { t.update(l, progress.Load(), err) }()
-
-				for _, err := range backoff.Loop(ctx, 3*time.Second) {
-					if err != nil {
-						return err
-					}
-					err := func() error {
-						req, err := http.NewRequestWithContext(ctx, "GET", cs.URL, nil)
-						if err != nil {
-							return err
-						}
-						req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", cs.Chunk.Start, cs.Chunk.End))
-						res, err := sendRequest(r.client(), req)
-						if err != nil {
-							return err
-						}
-						defer res.Body.Close()
-
-						// Count bytes towards
-						// progress, as they arrive, so
-						// that our bytes piggyback
-						// other chunk updates on
-						// completion.
-						//
-						// This tactic is enough to
-						// show "smooth" progress given
-						// the current CLI client. In
-						// the near future, the server
-						// should report download rate
-						// since it knows better than
-						// a client that is measuring
-						// rate based on wall-clock
-						// time-since-last-update.
-						body := &trackingReader{r: res.Body, n: &progress}
-
-						err = chunked.Put(cs.Chunk, cs.Digest, body)
-						if err != nil {
-							return err
-						}
-
-						return nil
-					}()
-					if !canRetry(err) {
-						return err
-					}
-				}
-				return nil
+				defer func() { q.CloseWithError(err) }()
+				return c.Put(l.Digest, q, l.Size)
 			})
+
+			var progress atomic.Int64
+
+			// We want to avoid extra round trips per chunk due to
+			// redirects from the registry to the blob store, so
+			// fire an initial request to get the final URL and
+			// then use that URL for the chunk requests.
+			req.Header.Set("Range", "bytes=0-0")
+			res, err := sendRequest(r.client(), req)
+			if err != nil {
+				return err
+			}
+			res.Body.Close()
+			req = res.Request.WithContext(req.Context())
+
+			wp := writerPool{size: r.maxChunkSize()}
+
+			for chunk := range chunks.Of(l.Size, r.maxChunkSize()) {
+				if ctx.Err() != nil {
+					break
+				}
+
+				ticket := q.Take()
+				g.Go(func() (err error) {
+					defer func() {
+						if err != nil {
+							q.CloseWithError(err)
+						}
+						ticket.Close()
+						t.update(l, progress.Load(), err)
+					}()
+
+					for _, err := range backoff.Loop(ctx, 3*time.Second) {
+						if err != nil {
+							return err
+						}
+						err := func() error {
+							req := req.Clone(req.Context())
+							req.Header.Set("Range", fmt.Sprintf("bytes=%s", chunk))
+							res, err := sendRequest(r.client(), req)
+							if err != nil {
+								return err
+							}
+							defer res.Body.Close()
+
+							tw := wp.get()
+							tw.Reset(ticket)
+							defer wp.put(tw)
+
+							_, err = io.CopyN(tw, res.Body, chunk.Size())
+							if err != nil {
+								return maybeUnexpectedEOF(err)
+							}
+							if err := tw.Flush(); err != nil {
+								return err
+							}
+
+							total := progress.Add(chunk.Size())
+							if total >= l.Size {
+								q.Close()
+							}
+							return nil
+						}()
+						if !canRetry(err) {
+							return err
+						}
+					}
+					return nil
+				})
+			}
 		}
 	}
+
 	if err := g.Wait(); err != nil {
 		return err
 	}
@@ -576,6 +615,8 @@ type Manifest struct {
 	Config *Layer `json:"config"`
 }
 
+var emptyDigest, _ = blob.ParseDigest("sha256:0000000000000000000000000000000000000000000000000000000000000000")
+
 // Layer returns the layer with the given
 // digest, or nil if not found.
 func (m *Manifest) Layer(d blob.Digest) *Layer {
@@ -602,9 +643,10 @@ func (m Manifest) MarshalJSON() ([]byte, error) {
 		// last phase of the commit which expects it, but does nothing
 		// with it. This will be fixed in a future release of
 		// ollama.com.
-		Config Layer `json:"config"`
+		Config *Layer `json:"config"`
 	}{
-		M: M(m),
+		M:      M(m),
+		Config: &Layer{Digest: emptyDigest},
 	}
 	return json.Marshal(v)
 }
@@ -694,123 +736,6 @@ func (r *Registry) Resolve(ctx context.Context, name string) (*Manifest, error)
 	return m, nil
 }
 
-type chunksum struct {
-	URL    string
-	Chunk  blob.Chunk
-	Digest blob.Digest
-}
-
-// chunksums returns a sequence of chunksums for the given layer. If the layer is under the
-// chunking threshold, a single chunksum is returned that covers the entire layer. If the layer
-// is over the chunking threshold, the chunksums are read from the chunksums endpoint.
-func (r *Registry) chunksums(ctx context.Context, name string, l *Layer) iter.Seq2[chunksum, error] {
-	return func(yield func(chunksum, error) bool) {
-		scheme, n, _, err := r.parseNameExtended(name)
-		if err != nil {
-			yield(chunksum{}, err)
-			return
-		}
-
-		if l.Size < r.maxChunkingThreshold() {
-			// any layer under the threshold should be downloaded
-			// in one go.
-			cs := chunksum{
-				URL: fmt.Sprintf("%s://%s/v2/%s/%s/blobs/%s",
-					scheme,
-					n.Host(),
-					n.Namespace(),
-					n.Model(),
-					l.Digest,
-				),
-				Chunk:  blob.Chunk{Start: 0, End: l.Size - 1},
-				Digest: l.Digest,
-			}
-			yield(cs, nil)
-			return
-		}
-
-		// A chunksums response is a sequence of chunksums in a
-		// simple, easy to parse line-oriented format.
-		//
-		// Example:
-		//
-		//     >> GET /v2/<namespace>/<model>/chunksums/<digest>
-		//
-		//     << HTTP/1.1 200 OK
-		//     << Content-Location: <blobURL>
-		//     <<
-		//     << <digest> <start>-<end>
-		//     << ...
-		//
-		// The blobURL is the URL to download the chunks from.
-
-		chunksumsURL := fmt.Sprintf("%s://%s/v2/%s/%s/chunksums/%s",
-			scheme,
-			n.Host(),
-			n.Namespace(),
-			n.Model(),
-			l.Digest,
-		)
-
-		req, err := r.newRequest(ctx, "GET", chunksumsURL, nil)
-		if err != nil {
-			yield(chunksum{}, err)
-			return
-		}
-		res, err := sendRequest(r.client(), req)
-		if err != nil {
-			yield(chunksum{}, err)
-			return
-		}
-		defer res.Body.Close()
-		if res.StatusCode != 200 {
-			err := fmt.Errorf("chunksums: unexpected status code %d", res.StatusCode)
-			yield(chunksum{}, err)
-			return
-		}
-		blobURL := res.Header.Get("Content-Location")
-
-		s := bufio.NewScanner(res.Body)
-		s.Split(bufio.ScanWords)
-		for {
-			if !s.Scan() {
-				if s.Err() != nil {
-					yield(chunksum{}, s.Err())
-				}
-				return
-			}
-			d, err := blob.ParseDigest(s.Bytes())
-			if err != nil {
-				yield(chunksum{}, fmt.Errorf("invalid digest: %q", s.Bytes()))
-				return
-			}
-
-			if !s.Scan() {
-				err := s.Err()
-				if err == nil {
-					err = fmt.Errorf("missing chunk range for digest %s", d)
-				}
-				yield(chunksum{}, err)
-				return
-			}
-			chunk, err := parseChunk(s.Bytes())
-			if err != nil {
-				yield(chunksum{}, fmt.Errorf("invalid chunk range for digest %s: %q", d, s.Bytes()))
-				return
-			}
-
-			cs := chunksum{
-				URL:    blobURL,
-				Chunk:  chunk,
-				Digest: d,
-			}
-			if !yield(cs, nil) {
-				return
-			}
-		}
-	}
-}
-
 func (r *Registry) client() *http.Client {
 	if r.HTTPClient != nil {
 		return r.HTTPClient
@@ -973,6 +898,13 @@ func checkData(url string) string {
 	return fmt.Sprintf("GET,%s,%s", url, zeroSum)
 }
 
+func maybeUnexpectedEOF(err error) error {
+	if errors.Is(err, io.EOF) {
+		return io.ErrUnexpectedEOF
+	}
+	return err
+}
+
 type publicError struct {
 	wrapped error
 	message string
@@ -1059,22 +991,27 @@ func splitExtended(s string) (scheme, name, digest string) {
 	return scheme, s, digest
 }
 
-// parseChunk parses a string in the form "start-end" and returns the Chunk.
-func parseChunk[S ~string | ~[]byte](s S) (blob.Chunk, error) {
-	startPart, endPart, found := strings.Cut(string(s), "-")
-	if !found {
-		return blob.Chunk{}, fmt.Errorf("chunks: invalid range %q: missing '-'", s)
-	}
-	start, err := strconv.ParseInt(startPart, 10, 64)
-	if err != nil {
-		return blob.Chunk{}, fmt.Errorf("chunks: invalid start to %q: %v", s, err)
-	}
-	end, err := strconv.ParseInt(endPart, 10, 64)
-	if err != nil {
-		return blob.Chunk{}, fmt.Errorf("chunks: invalid end to %q: %v", s, err)
-	}
-	if start > end {
-		return blob.Chunk{}, fmt.Errorf("chunks: invalid range %q: start > end", s)
-	}
-	return blob.Chunk{Start: start, End: end}, nil
+type writerPool struct {
+	size int64 // set by the caller
+
+	mu sync.Mutex
+	ws []*bufio.Writer
+}
+
+func (p *writerPool) get() *bufio.Writer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if len(p.ws) == 0 {
+		return bufio.NewWriterSize(nil, int(p.size))
+	}
+	w := p.ws[len(p.ws)-1]
+	p.ws = p.ws[:len(p.ws)-1]
+	return w
+}
+
+func (p *writerPool) put(w *bufio.Writer) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	w.Reset(nil)
+	p.ws = append(p.ws, w)
 }

server/internal/client/ollama/registry_test.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/ollama/ollama/server/internal/cache/blob"
+	"github.com/ollama/ollama/server/internal/chunks"
 	"github.com/ollama/ollama/server/internal/testutil"
 )
 
@@ -427,7 +428,7 @@ func TestRegistryPullCached(t *testing.T) {
 	err := rc.Pull(ctx, "single")
 	testutil.Check(t, err)
 
-	want := []int64{0, 6}
+	want := []int64{6}
 	if !errors.Is(errors.Join(errs...), ErrCached) {
 		t.Errorf("errs = %v; want %v", errs, ErrCached)
 	}
@@ -530,6 +531,54 @@ func TestRegistryPullMixedCachedNotCached(t *testing.T) {
 	}
 }
 
+func TestRegistryPullChunking(t *testing.T) {
+	rc, _ := newClient(t, func(w http.ResponseWriter, r *http.Request) {
+		t.Log("request:", r.URL.Host, r.Method, r.URL.Path, r.Header.Get("Range"))
+		if r.URL.Host != "blob.store" {
+			// The production registry redirects to the blob store.
+			http.Redirect(w, r, "http://blob.store"+r.URL.Path, http.StatusFound)
+			return
+		}
+		if strings.Contains(r.URL.Path, "/blobs/") {
+			rng := r.Header.Get("Range")
+			if rng == "" {
+				http.Error(w, "missing range", http.StatusBadRequest)
+				return
+			}
+			_, c, err := chunks.ParseRange(r.Header.Get("Range"))
+			if err != nil {
+				panic(err)
+			}
+			io.WriteString(w, "remote"[c.Start:c.End+1])
+			return
+		}
+		fmt.Fprintf(w, `{"layers":[{"digest":%q,"size":6}]}`, blob.DigestFromBytes("remote"))
+	})
+
+	// Force chunking by setting the threshold to less than the size of the
+	// layer.
+	rc.ChunkingThreshold = 3
+	rc.MaxChunkSize = 3
+
+	var reads []int64
+	ctx := WithTrace(t.Context(), &Trace{
+		Update: func(d *Layer, n int64, err error) {
+			if err != nil {
+				t.Errorf("update %v %d %v", d, n, err)
+			}
+			reads = append(reads, n)
+		},
+	})
+
+	err := rc.Pull(ctx, "remote")
+	testutil.Check(t, err)
+
+	want := []int64{0, 3, 6}
+	if !slices.Equal(reads, want) {
+		t.Errorf("reads = %v; want %v", reads, want)
+	}
+}
+
 func TestRegistryResolveByDigest(t *testing.T) {
 	check := testutil.Checker(t)
 

server/internal/cmd/oppbench/oppbench.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	fmt.Println("Run as 'go test -bench=.' to run the benchmarks")
+	os.Exit(1)
+}

server/internal/cmd/oppbench/oppbench_test.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/ollama/ollama/server/internal/chunks"
+	"golang.org/x/sync/errgroup"
+)
+
+func BenchmarkDownload(b *testing.B) {
+	run := func(fileSize, chunkSize int64) {
+		name := fmt.Sprintf("size=%d/chunksize=%d", fileSize, chunkSize)
+		b.Run(name, func(b *testing.B) { benchmarkDownload(b, fileSize, chunkSize) })
+	}
+
+	run(100<<20, 8<<20)
+	run(100<<20, 16<<20)
+	run(100<<20, 32<<20)
+	run(100<<20, 64<<20)
+	run(100<<20, 128<<20) // 1 chunk
+}
+
+func run(ctx context.Context, c *http.Client, chunk chunks.Chunk) error {
+	const blobURL = "https://ollama.com/v2/x/x/blobs/sha256-4824460d29f2058aaf6e1118a63a7a197a09bed509f0e7d4e2efb1ee273b447d"
+	req, err := http.NewRequestWithContext(ctx, "GET", blobURL, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Range", fmt.Sprintf("bytes=%s", chunk))
+	res, err := c.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	_, err = io.CopyN(io.Discard, res.Body, chunk.Size()) // will io.EOF on short read
+	return err
+}
+
+var sleepTime atomic.Int64
+
+func benchmarkDownload(b *testing.B, fileSize, chunkSize int64) {
+	client := &http.Client{
+		Transport: func() http.RoundTripper {
+			tr := http.DefaultTransport.(*http.Transport).Clone()
+			tr.DisableKeepAlives = true
+			return tr
+		}(),
+	}
+	defer client.CloseIdleConnections()
+
+	// warm up the client
+	run(context.Background(), client, chunks.New(0, 1<<20))
+
+	b.SetBytes(fileSize)
+	b.ReportAllocs()
+
+	// Give our CDN a min to breathe between benchmarks.
+	time.Sleep(time.Duration(sleepTime.Swap(3)))
+
+	for b.Loop() {
+		g, ctx := errgroup.WithContext(b.Context())
+		g.SetLimit(runtime.GOMAXPROCS(0))
+		for chunk := range chunks.Of(fileSize, chunkSize) {
+			g.Go(func() error { return run(ctx, client, chunk) })
+		}
+		if err := g.Wait(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkWrite(b *testing.B) {
+	b.Run("chunksize=1MiB", func(b *testing.B) { benchmarkWrite(b, 1<<20) })
+}
+
+func benchmarkWrite(b *testing.B, chunkSize int) {
+	b.ReportAllocs()
+
+	dir := b.TempDir()
+	f, err := os.Create(filepath.Join(dir, "write-single"))
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer f.Close()
+
+	data := make([]byte, chunkSize)
+	b.SetBytes(int64(chunkSize))
+	r := bytes.NewReader(data)
+	for b.Loop() {
+		r.Reset(data)
+		_, err := io.Copy(f, r)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

server/internal/registry/server.go

@@ -1,5 +1,6 @@
-// Package registry implements an http.Handler for handling local Ollama API
-// model management requests. See [Local] for details.
+// Package registry provides an http.Handler for handling local Ollama API
+// requests for performing tasks related to the ollama.com model registry and
+// the local disk cache.
 package registry
 
 import (
@@ -9,7 +10,6 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
-	"maps"
 	"net/http"
 	"sync"
 	"time"
@@ -18,11 +18,16 @@ import (
 	"github.com/ollama/ollama/server/internal/client/ollama"
 )
 
-// Local implements an http.Handler for handling local Ollama API model
-// management requests, such as pushing, pulling, and deleting models.
+// Local is an http.Handler for handling local Ollama API requests for
+// performing tasks related to the ollama.com model registry combined with the
+// local disk cache.
 //
-// It can be arranged for all unknown requests to be passed through to a
-// fallback handler, if one is provided.
+// It is not concern of Local, or this package, to handle model creation, which
+// proceeds any registry operations for models it produces.
+//
+// NOTE: The package built for dealing with model creation should use
+// [DefaultCache] to access the blob store and not attempt to read or write
+// directly to the blob disk cache.
 type Local struct {
 	Client *ollama.Registry // required
 	Logger *slog.Logger     // required
@@ -58,7 +63,6 @@ func (e serverError) Error() string {
 var (
 	errMethodNotAllowed = &serverError{405, "method_not_allowed", "method not allowed"}
 	errNotFound         = &serverError{404, "not_found", "not found"}
-	errModelNotFound    = &serverError{404, "not_found", "model not found"}
 	errInternalError    = &serverError{500, "internal_error", "internal server error"}
 )
 
@@ -171,16 +175,8 @@ func (s *Local) serveHTTP(rec *statusCodeRecorder, r *http.Request) {
 }
 
 type params struct {
-	// DeprecatedName is the name of the model to push, pull, or delete,
-	// but is deprecated. New clients should use [Model] instead.
-	//
-	// Use [model()] to get the model name for both old and new API requests.
-	DeprecatedName string `json:"name"`
-
-	// Model is the name of the model to push, pull, or delete.
-	//
-	// Use [model()] to get the model name for both old and new API requests.
-	Model string `json:"model"`
+	DeprecatedName string `json:"name"`  // Use [params.model]
+	Model          string `json:"model"` // Use [params.model]
 
 	// AllowNonTLS is a flag that indicates a client using HTTP
 	// is doing so, deliberately.
@@ -193,18 +189,9 @@ type params struct {
 	// confusing flags such as this.
 	AllowNonTLS bool `json:"insecure"`
 
-	// Stream, if true, will make the server send progress updates in a
-	// streaming of JSON objects. If false, the server will send a single
-	// JSON object with the final status as "success", or an error object
-	// if an error occurred.
-	//
-	// Unfortunately, this API was designed to be a bit awkward. Stream is
-	// defined to default to true if not present, so we need a way to check
-	// if the client decisively it to false. So, we use a pointer to a
-	// bool. Gross.
-	//
-	// Use [stream()] to get the correct value for this field.
-	Stream *bool `json:"stream"`
+	// ProgressStream is a flag that indicates the client is expecting a stream of
+	// progress updates.
+	ProgressStream bool `json:"stream"`
 }
 
 // model returns the model name for both old and new API requests.
@@ -212,13 +199,6 @@ func (p params) model() string {
 	return cmp.Or(p.Model, p.DeprecatedName)
 }
 
-func (p params) stream() bool {
-	if p.Stream == nil {
-		return true
-	}
-	return *p.Stream
-}
-
 func (s *Local) handleDelete(_ http.ResponseWriter, r *http.Request) error {
 	if r.Method != "DELETE" {
 		return errMethodNotAllowed
@@ -232,16 +212,16 @@ func (s *Local) handleDelete(_ http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 	if !ok {
-		return errModelNotFound
+		return &serverError{404, "not_found", "model not found"}
 	}
-	if s.Prune != nil {
-		return s.Prune()
+	if s.Prune == nil {
+		return nil
 	}
-	return nil
+	return s.Prune()
 }
 
 type progressUpdateJSON struct {
-	Status    string      `json:"status,omitempty,omitzero"`
+	Status    string      `json:"status"`
 	Digest    blob.Digest `json:"digest,omitempty,omitzero"`
 	Total     int64       `json:"total,omitempty,omitzero"`
 	Completed int64       `json:"completed,omitempty,omitzero"`
@@ -257,17 +237,6 @@ func (s *Local) handlePull(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
-	enc := json.NewEncoder(w)
-	if !p.stream() {
-		if err := s.Client.Pull(r.Context(), p.model()); err != nil {
-			if errors.Is(err, ollama.ErrModelNotFound) {
-				return errModelNotFound
-			}
-			return err
-		}
-		return enc.Encode(progressUpdateJSON{Status: "success"})
-	}
-
 	maybeFlush := func() {
 		fl, _ := w.(http.Flusher)
 		if fl != nil {
@@ -277,67 +246,69 @@ func (s *Local) handlePull(w http.ResponseWriter, r *http.Request) error {
 	defer maybeFlush()
 
 	var mu sync.Mutex
-	progress := make(map[*ollama.Layer]int64)
+	enc := json.NewEncoder(w)
+	enc.Encode(progressUpdateJSON{Status: "pulling manifest"})
 
-	progressCopy := make(map[*ollama.Layer]int64, len(progress))
-	pushUpdate := func() {
-		defer maybeFlush()
+	ctx := ollama.WithTrace(r.Context(), &ollama.Trace{
+		Update: func(l *ollama.Layer, n int64, err error) {
+			mu.Lock()
+			defer mu.Unlock()
 
-		// TODO(bmizerany): This scales poorly with more layers due to
-		// needing to flush out them all in one big update. We _could_
-		// just flush on the changed ones, or just track the whole
-		// download. Needs more thought. This is fine for now.
-		mu.Lock()
-		maps.Copy(progressCopy, progress)
-		mu.Unlock()
-		for l, n := range progress {
+			// TODO(bmizerany): coalesce these updates; writing per
+			// update is expensive
 			enc.Encode(progressUpdateJSON{
 				Digest:    l.Digest,
+				Status:    "pulling",
 				Total:     l.Size,
 				Completed: n,
 			})
-		}
-	}
-
-	t := time.NewTicker(time.Hour) // "unstarted" timer
-	start := sync.OnceFunc(func() {
-		pushUpdate()
-		t.Reset(100 * time.Millisecond)
-	})
-	ctx := ollama.WithTrace(r.Context(), &ollama.Trace{
-		Update: func(l *ollama.Layer, n int64, err error) {
-			if n > 0 {
-				start() // flush initial state
-			}
-			mu.Lock()
-			progress[l] = n
-			mu.Unlock()
 		},
 	})
 
 	done := make(chan error, 1)
 	go func() {
+		// TODO(bmizerany): continue to support non-streaming responses
 		done <- s.Client.Pull(ctx, p.model())
 	}()
 
-	for {
-		select {
-		case <-t.C:
-			pushUpdate()
-		case err := <-done:
-			pushUpdate()
-			if err != nil {
-				var status string
-				if errors.Is(err, ollama.ErrModelNotFound) {
-					status = fmt.Sprintf("error: model %q not found", p.model())
-				} else {
-					status = fmt.Sprintf("error: %v", err)
+	func() {
+		t := time.NewTicker(100 * time.Millisecond)
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				mu.Lock()
+				maybeFlush()
+				mu.Unlock()
+			case err := <-done:
+				if err != nil {
+					var status string
+					if errors.Is(err, ollama.ErrModelNotFound) {
+						status = fmt.Sprintf("error: model %q not found", p.model())
+						enc.Encode(progressUpdateJSON{Status: status})
+					} else {
+						status = fmt.Sprintf("error: %v", err)
+						enc.Encode(progressUpdateJSON{Status: status})
+					}
+					return
 				}
-				enc.Encode(progressUpdateJSON{Status: status})
+
+				// These final updates are not strictly necessary, because they have
+				// already happened at this point. Our pull handler code used to do
+				// these steps after, not during, the pull, and they were slow, so we
+				// wanted to provide feedback to users what was happening. For now, we
+				// keep them to not jar users who are used to seeing them. We can phase
+				// them out with a new and nicer UX later. One without progress bars
+				// and digests that no one cares about.
+				enc.Encode(progressUpdateJSON{Status: "verifying layers"})
+				enc.Encode(progressUpdateJSON{Status: "writing manifest"})
+				enc.Encode(progressUpdateJSON{Status: "success"})
+				return
 			}
-			return nil
 		}
-	}
+	}()
+
+	return nil
 }
 
 func decodeUserJSON[T any](r io.Reader) (T, error) {

server/internal/registry/server_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/fs"
 	"net"
@@ -159,6 +160,7 @@ var registryFS = sync.OnceValue(func() fs.FS {
 	// to \n when parsing the txtar on Windows.
 	data := bytes.ReplaceAll(registryTXT, []byte("\r\n"), []byte("\n"))
 	a := txtar.Parse(data)
+	fmt.Printf("%q\n", a.Comment)
 	fsys, err := txtar.FS(a)
 	if err != nil {
 		panic(err)
@@ -177,7 +179,7 @@ func TestServerPull(t *testing.T) {
 			w.WriteHeader(404)
 			io.WriteString(w, `{"errors": [{"code": "MANIFEST_UNKNOWN", "message": "manifest unknown"}]}`)
 		default:
-			t.Logf("serving blob: %s", r.URL.Path)
+			t.Logf("serving file: %s", r.URL.Path)
 			modelsHandler.ServeHTTP(w, r)
 		}
 	})
@@ -186,7 +188,7 @@ func TestServerPull(t *testing.T) {
 		t.Helper()
 
 		if got.Code != 200 {
-			t.Errorf("Code = %d; want 200", got.Code)
+			t.Fatalf("Code = %d; want 200", got.Code)
 		}
 		gotlines := got.Body.String()
 		t.Logf("got:\n%s", gotlines)
@@ -195,29 +197,35 @@ func TestServerPull(t *testing.T) {
 			want, unwanted := strings.CutPrefix(want, "!")
 			want = strings.TrimSpace(want)
 			if !unwanted && !strings.Contains(gotlines, want) {
-				t.Errorf("! missing %q in body", want)
+				t.Fatalf("! missing %q in body", want)
 			}
 			if unwanted && strings.Contains(gotlines, want) {
-				t.Errorf("! unexpected %q in body", want)
+				t.Fatalf("! unexpected %q in body", want)
 			}
 		}
 	}
 
 	got := s.send(t, "POST", "/api/pull", `{"model": "BOOM"}`)
 	checkResponse(got, `
+		{"status":"pulling manifest"}
 		{"status":"error: request error https://example.com/v2/library/BOOM/manifests/latest: registry responded with status 999: boom"}
 	`)
 
 	got = s.send(t, "POST", "/api/pull", `{"model": "smol"}`)
 	checkResponse(got, `
-		{"digest":"sha256:68e0ec597aee59d35f8dc44942d7b17d471ade10d3aca07a5bb7177713950312","total":5}
-		{"digest":"sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","total":3}
-		{"digest":"sha256:68e0ec597aee59d35f8dc44942d7b17d471ade10d3aca07a5bb7177713950312","total":5,"completed":5}
-		{"digest":"sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","total":3,"completed":3}
+		{"status":"pulling manifest"}
+		{"status":"pulling","digest":"sha256:68e0ec597aee59d35f8dc44942d7b17d471ade10d3aca07a5bb7177713950312","total":5}
+		{"status":"pulling","digest":"sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","total":3}
+		{"status":"pulling","digest":"sha256:68e0ec597aee59d35f8dc44942d7b17d471ade10d3aca07a5bb7177713950312","total":5,"completed":5}
+		{"status":"pulling","digest":"sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","total":3,"completed":3}
+		{"status":"verifying layers"}
+		{"status":"writing manifest"}
+		{"status":"success"}
 	`)
 
 	got = s.send(t, "POST", "/api/pull", `{"model": "unknown"}`)
 	checkResponse(got, `
+		{"status":"pulling manifest"}
 		{"status":"error: model \"unknown\" not found"}
 	`)
 
@@ -232,39 +240,19 @@ func TestServerPull(t *testing.T) {
 
 	got = s.send(t, "POST", "/api/pull", `{"model": "://"}`)
 	checkResponse(got, `
+		{"status":"pulling manifest"}
 		{"status":"error: invalid or missing name: \"\""}
-	`)
 
-	// Non-streaming pulls
-	got = s.send(t, "POST", "/api/pull", `{"model": "://", "stream": false}`)
-	checkErrorResponse(t, got, 400, "bad_request", "invalid or missing name")
-	got = s.send(t, "POST", "/api/pull", `{"model": "smol", "stream": false}`)
-	checkResponse(got, `
-		{"status":"success"}
-		!digest
-		!total
-		!completed
+		!verifying
+		!writing
+		!success
 	`)
-	got = s.send(t, "POST", "/api/pull", `{"model": "unknown", "stream": false}`)
-	checkErrorResponse(t, got, 404, "not_found", "model not found")
 }
 
 func TestServerUnknownPath(t *testing.T) {
 	s := newTestServer(t, nil)
 	got := s.send(t, "DELETE", "/api/unknown", `{}`)
 	checkErrorResponse(t, got, 404, "not_found", "not found")
-
-	var fellback bool
-	s.Fallback = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fellback = true
-	})
-	got = s.send(t, "DELETE", "/api/unknown", `{}`)
-	if !fellback {
-		t.Fatal("expected Fallback to be called")
-	}
-	if got.Code != 200 {
-		t.Fatalf("Code = %d; want 200", got.Code)
-	}
 }
 
 func checkErrorResponse(t *testing.T, got *httptest.ResponseRecorder, status int, code, msg string) {

server/prompt.go

@@ -26,7 +26,6 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.
 	var system []api.Message
 
 	isMllama := checkMllamaModelFamily(m)
-	isGemma3 := checkGemma3ModelFamily(m)
 
 	var imageNumTokens int
 	// TODO: Ideally we would compute this from the projector metadata but some pieces are implementation dependent
@@ -41,7 +40,7 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.
 	n := len(msgs) - 1
 	// in reverse, find all messages that fit into context window
 	for i := n; i >= 0; i-- {
-		if (isMllama || isGemma3) && len(msgs[i].Images) > 1 {
+		if isMllama && len(msgs[i].Images) > 1 {
 			return "", nil, errTooManyImages
 		}
 
@@ -158,12 +157,3 @@ func checkMllamaModelFamily(m *Model) bool {
 	}
 	return false
 }
-
-func checkGemma3ModelFamily(m *Model) bool {
-	for _, arch := range m.Config.ModelFamilies {
-		if arch == "gemma3" {
-			return true
-		}
-	}
-	return false
-}

server/routes.go

@@ -435,7 +435,7 @@ func (s *Server) EmbedHandler(c *gin.Context) {
 		return
 	}
 
-	kvData, _, err := getModelData(m.ModelPath, false)
+	kvData, err := getKVData(m.ModelPath, false)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
@@ -483,7 +483,8 @@ func (s *Server) EmbedHandler(c *gin.Context) {
 	}
 
 	if err := g.Wait(); err != nil {
-		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": strings.TrimSpace(err.Error())})
+		slog.Error("embedding generation failed", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embeddings: %v", err)})
 		return
 	}
 
@@ -544,7 +545,8 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) {
 
 	embedding, err := r.Embedding(c.Request.Context(), req.Prompt)
 	if err != nil {
-		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": strings.TrimSpace(err.Error())})
+		slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
+		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embedding: %v", err)})
 		return
 	}
 
@@ -848,23 +850,16 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
 	fmt.Fprint(&sb, m.String())
 	resp.Modelfile = sb.String()
 
-	kvData, tensors, err := getModelData(m.ModelPath, req.Verbose)
+	kvData, err := getKVData(m.ModelPath, req.Verbose)
 	if err != nil {
 		return nil, err
 	}
-
 	delete(kvData, "general.name")
 	delete(kvData, "tokenizer.chat_template")
 	resp.ModelInfo = kvData
 
-	tensorData := make([]api.Tensor, len(tensors.Items()))
-	for cnt, t := range tensors.Items() {
-		tensorData[cnt] = api.Tensor{Name: t.Name, Type: t.Type(), Shape: t.Shape}
-	}
-	resp.Tensors = tensorData
-
 	if len(m.ProjectorPaths) > 0 {
-		projectorData, _, err := getModelData(m.ProjectorPaths[0], req.Verbose)
+		projectorData, err := getKVData(m.ProjectorPaths[0], req.Verbose)
 		if err != nil {
 			return nil, err
 		}
@@ -874,17 +869,17 @@ func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
 	return resp, nil
 }
 
-func getModelData(digest string, verbose bool) (ggml.KV, ggml.Tensors, error) {
+func getKVData(digest string, verbose bool) (ggml.KV, error) {
 	maxArraySize := 0
 	if verbose {
 		maxArraySize = -1
 	}
-	data, err := llm.LoadModel(digest, maxArraySize)
+	kvData, err := llm.LoadModel(digest, maxArraySize)
 	if err != nil {
-		return nil, ggml.Tensors{}, err
+		return nil, err
 	}
 
-	kv := data.KV()
+	kv := kvData.KV()
 
 	if !verbose {
 		for k := range kv {
@@ -894,7 +889,7 @@ func getModelData(digest string, verbose bool) (ggml.KV, ggml.Tensors, error) {
 		}
 	}
 
-	return kv, data.Tensors(), nil
+	return kv, nil
 }
 
 func (s *Server) ListHandler(c *gin.Context) {

server/sched.go

@@ -194,11 +194,6 @@ func (s *Scheduler) processPending(ctx context.Context) {
 						break
 					}
 
-					// Embedding models should always be loaded with parallel=1
-					if pending.model.CheckCapabilities(CapabilityCompletion) != nil {
-						numParallel = 1
-					}
-
 					// Evaluate if the model will fit in the available system memory, or if we should unload a model first
 					if len(gpus) == 1 && gpus[0].Library == "cpu" {
 						// simplifying assumption of defaultParallel when in CPU mode