ollama/create/classify.go

package create

import (
	"fmt"
	"strings"

	"github.com/ollama/ollama/mlx/quant"
)

// SourceKind is the overarching dtype for a given safetensors model
type SourceKind int

const (
	SourceFloat        SourceKind = iota // bf16/fp16/fp32 — quantizable on request
	SourceBlockFP8                       // HF block-FP8 — auto-converted to mxfp8
	SourcePrequantized                   // already quantized — copied through
)

func (k SourceKind) String() string {
	switch k {
	case SourceFloat:
		return "float"
	case SourceBlockFP8:
		return "block-fp8"
	case SourcePrequantized:
		return "prequantized"
	default:
		return "unknown"
	}
}

// Classification is the decision about a source model: its kind and the
// effective quantization of the imported weights. Quantize may describe a
// requested conversion or a quantization already present in the source.
type Classification struct {
	Kind     SourceKind
	Quantize string
}

// Classify decides a source model's kind and resolves the effective
// quantization from the user's requested type, rejecting requests that are not
// allowed for the kind.
func Classify(inv Inventory, requested string) (Classification, error) {
	requested, err := normalizeRequested(requested)
	if err != nil {
		return Classification{}, err
	}

	if name, ok := firstUnsupportedFP8(inv); ok {
		return Classification{}, fmt.Errorf("unsupported fp8 source: tensor %s is F8_E5M2; only F8_E4M3 block-FP8 sources are supported", name)
	}

	switch detectKind(inv) {
	case SourceFloat:
		return Classification{Kind: SourceFloat, Quantize: requested}, nil

	case SourcePrequantized:
		if requested != "" {
			return Classification{}, fmt.Errorf("cannot requantize an already-quantized source model (requested %q): only bf16/fp16/fp32 sources can be quantized", requested)
		}
		return Classification{
			Kind:     SourcePrequantized,
			Quantize: detectPrequantizedQuantization(inv),
		}, nil

	case SourceBlockFP8:
		rows, cols, ok := inv.Config.HFFP8WeightBlockSize()
		if !ok {
			return Classification{}, fmt.Errorf("fp8 source model is missing weight_block_size metadata")
		}
		if rows != 128 || cols != 128 {
			return Classification{}, fmt.Errorf("unsupported fp8 source block size %dx%d (only 128x128 is supported)", rows, cols)
		}
		if requested != "" {
			return Classification{}, fmt.Errorf("cannot quantize an fp8 source model (requested %q): fp8 sources are converted to mxfp8 automatically; only bf16/fp16/fp32 sources can be quantized", requested)
		}
		return Classification{Kind: SourceBlockFP8, Quantize: "mxfp8"}, nil
	}

	return Classification{}, fmt.Errorf("could not classify source model in %s", inv.Dir)
}

// detectPrequantizedQuantization reports a single quantization shared by the
// source's recognized prequantized weights. A mixed or unrecognized source has
// no single file type to record in the model manifest.
func detectPrequantizedQuantization(inv Inventory) string {
	var detected string
	for _, name := range sortedTensorNames(inv) {
		spec, _, ok := matchPrequant(name, inv)
		if !ok {
			continue
		}
		q := quant.Canonical(spec.Metadata["quant_type"])
		if q == "" {
			continue
		}
		if detected != "" && detected != q {
			return ""
		}
		detected = q
	}
	return detected
}

// normalizeRequested validates the user's quantize value and returns its
// canonical form ("" for no quantization).
func normalizeRequested(requested string) (string, error) {
	if strings.TrimSpace(requested) == "" {
		return "", nil
	}
	c := quant.Canonical(requested)
	if c == "" {
		return "", fmt.Errorf("unsupported quantize type %q: supported types are int4, int8, nvfp4, mxfp4, mxfp8", requested)
	}
	return c, nil
}

// detectKind sorts a source into Float, BlockFP8, or Prequantized using only
// the inventory's tensor names, dtypes, and config. Prequantized is detected
// from the tensors themselves, so a model whose quantization config sidecar is
// missing (e.g. a ModelOpt checkpoint without hf_quant_config.json) is still
// recognized as already-quantized and not mistaken for a float model.
func detectKind(inv Inventory) SourceKind {
	var hasMLXScales, hasPacked, hasNVFP4Scale, hasFP8Weight bool
	for name, t := range inv.Tensors {
		switch {
		case strings.HasSuffix(name, ".scales"):
			hasMLXScales = true
		case strings.HasSuffix(name, ".weight_packed"):
			hasPacked = true
		case strings.HasSuffix(name, ".weight_scale"):
			// An NVFP4 per-block scale sits on a packed (U8) weight. A
			// block-FP8 source also has a scale companion, but its weight is
			// F8_E4M3 — so the base weight's dtype disambiguates the two.
			if bt, ok := inv.Tensors[strings.TrimSuffix(name, "_scale")]; ok && isPackedDtype(bt.Dtype) {
				hasNVFP4Scale = true
			}
		}
		if strings.HasSuffix(name, ".weight") && isE4M3Dtype(t.Dtype) {
			hasFP8Weight = true
		}
	}

	switch {
	case hasMLXScales || hasPacked || hasNVFP4Scale:
		return SourcePrequantized
	case hasFP8Weight:
		return SourceBlockFP8
	default:
		return SourceFloat
	}
}

// firstUnsupportedFP8 returns the name of the first F8_E5M2 weight in the
// source, if any. We decode only E4M3, so an E5M2 source must be rejected
// explicitly rather than silently mishandled.
func firstUnsupportedFP8(inv Inventory) (string, bool) {
	for name, t := range inv.Tensors {
		if strings.HasSuffix(name, ".weight") && isE5M2Dtype(t.Dtype) {
			return name, true
		}
	}
	return "", false
}

func isPackedDtype(dtype string) bool {
	switch strings.ToUpper(dtype) {
	case "U8", "U32": // current .weight_scale producers ship U8; U32 covers a future word-packed source
		return true
	default:
		return false
	}
}

func isE4M3Dtype(dtype string) bool {
	switch strings.ToUpper(dtype) {
	case "F8_E4M3", "F8_E4M3FN":
		return true
	default:
		return false
	}
}

func isE5M2Dtype(dtype string) bool {
	switch strings.ToUpper(dtype) {
	case "F8_E5M2", "F8_E5M2FNUZ":
		return true
	default:
		return false
	}
}