From 7159c381ea998f7fe803635d247622b23859ce1a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 22:32:04 +0000 Subject: [PATCH] gallery: size model variants with a live probe, drop the nightly denormalizer Selection needs each variant's size to decide whether it fits and to rank largest-first. That figure was written into the index by a nightly job, which made the gallery carry a derived value that could drift from the entry it was derived from. Derive it at install time instead. pkg/vram already sizes a model without downloading it, and the gallery UI already uses it: a remote GGUF header range-fetch, then an HTTP HEAD for the content length, then any declared size:. It caches its results, so reuse it rather than writing a second probing path. A probe failure must never fail an install, so an unprobeable variant is treated as unknown: it survives the memory filter, because nothing proves it does not fit, and it ranks last, so a known-good fit always beats a guess. If every probe fails, selection still terminates on the base entry. The probe is injected through ResolveEnv rather than called directly, for the same reason the backend compatibility check is: specs pin an exact size, or an exact failure, without reaching the network. With that in place three things are dead weight and go: - The nightly job and the fields it populated. Variant.Backend was redundant because the backend is resolved live from the referenced entry during selection, and Quantization was display-only that nothing read. - min_memory on the base entry. The base always installs and its floor could only warn, so it could not change any outcome. - The lint rules and schema entries for both. min_memory on individual variants stays, as the override for when the probed size is wrong. An authored figure now suppresses the probe entirely rather than merely outranking it, so it costs no round trip. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .github/ci/gallery_denormalize.go | 296 --------------------- .github/workflows/gallery-denormalize.yaml | 56 ---- core/gallery/models.go | 86 +++++- core/gallery/models_types.go | 4 - core/gallery/resolve_variant.go | 41 ++- core/gallery/resolve_variant_test.go | 110 +++++--- core/gallery/variant.go | 34 +-- core/gallery/variants_install_test.go | 109 +++++++- core/gallery/variants_lint_test.go | 66 ++--- core/schema/gallery-model.schema.json | 18 +- gallery/index.yaml | 8 +- 11 files changed, 326 insertions(+), 502 deletions(-) delete mode 100644 .github/ci/gallery_denormalize.go delete mode 100644 .github/workflows/gallery-denormalize.yaml diff --git a/.github/ci/gallery_denormalize.go b/.github/ci/gallery_denormalize.go deleted file mode 100644 index 2dd0df909..000000000 --- a/.github/ci/gallery_denormalize.go +++ /dev/null @@ -1,296 +0,0 @@ -// Command gallery_denormalize fills the read-only denormalized fields on the -// variants of gallery model entries: backend, quantization, and -// inferred_min_memory. -// -// It never modifies an authored min_memory. Authored values are authoritative, -// because a human who measured a real load knows more than a pre-download -// estimate does. -package main - -import ( - "bytes" - "context" - "fmt" - "os" - "time" - - "gopkg.in/yaml.v3" - - "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/pkg/vram" -) - -const estimateContext = 8192 - -func main() { - if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "usage: gallery_denormalize ") - os.Exit(1) - } - path := os.Args[1] - - data, err := os.ReadFile(path) - if err != nil { - fmt.Fprintf(os.Stderr, "read %s: %v\n", path, err) - os.Exit(1) - } - - var entries []gallery.GalleryModel - if err := yaml.Unmarshal(data, &entries); err != nil { - fmt.Fprintf(os.Stderr, "parse %s: %v\n", path, err) - os.Exit(1) - } - - byName := map[string]gallery.GalleryModel{} - for _, e := range entries { - byName[e.Name] = e - } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - failures := 0 - for i := range entries { - if !entries[i].HasVariants() { - continue - } - for j := range entries[i].Variants { - c := &entries[i].Variants[j] - - target, ok := byName[c.Model] - if !ok { - fmt.Fprintf(os.Stderr, "%s: variant %q not found\n", entries[i].Name, c.Model) - failures++ - continue - } - - // The referenced entry's payload usually lives behind its url - // rather than inline, so fetch it. Metadata.Backend is populated - // at load time by the running server, not by parsing the index, - // so it is always empty here and cannot be copied. - resolved, err := gallery.GetGalleryConfigFromURLWithContext[gallery.ModelConfig](ctx, target.URL, "") - if err != nil { - fmt.Fprintf(os.Stderr, "%s: cannot fetch config for %q: %v\n", entries[i].Name, c.Model, err) - failures++ - continue - } - - c.Backend = backendOf(target, resolved) - c.Quantization = quantizationOf(target) - - // An authored value wins outright, so it gets no inferred - // counterpart. Clearing first matters: a variant that gained an - // authored min_memory would otherwise keep a stale inferred figure - // that EffectiveMinMemory would go on reporting once the authored - // one is removed again. - if c.MinMemory != "" { - c.InferredMinMemory = "" - continue - } - - // Weight files can come from either side: a referenced index entry - // usually lists them itself (files:, i.e. AdditionalFiles) while - // the url it points at is only a base config, but some entries - // invert that. Install time downloads both, so estimate over both. - files := make([]vram.FileInput, 0, len(target.AdditionalFiles)+len(resolved.Files)) - for _, f := range target.AdditionalFiles { - files = append(files, vram.FileInput{URI: f.URI}) - } - for _, f := range resolved.Files { - files = append(files, vram.FileInput{URI: f.URI}) - } - - estimate, err := vram.EstimateModelMultiContext(ctx, vram.ModelEstimateInput{ - Files: files, - Size: target.Size, - }, []uint32{estimateContext}) - if err != nil || estimate.VRAMForContext(estimateContext) == 0 { - fmt.Fprintf(os.Stderr, - "%s: could not estimate min_memory for %q (%v); author one by hand\n", - entries[i].Name, c.Model, err) - failures++ - continue - } - c.InferredMinMemory = fmt.Sprintf("%dMiB", estimate.VRAMForContext(estimateContext)/(1024*1024)) - } - } - - if err := writeVariants(path, data, entries); err != nil { - fmt.Fprintf(os.Stderr, "write %s: %v\n", path, err) - os.Exit(1) - } - - if failures > 0 { - fmt.Fprintf(os.Stderr, "%d variant(s) need a hand-authored min_memory\n", failures) - os.Exit(1) - } -} - -// writeVariants writes back only the three derived keys on each variant, -// leaving every other byte of the index alone. -// -// Round-tripping through []GalleryModel would reflow all 1200+ entries -// (quoting, key order, line wrapping), burying the handful of real changes in -// reformatting noise and making the nightly PR unreviewable. Even re-encoding -// just the variants subtree would restyle authored fields such as min_memory, -// so the rewrite reaches down to individual mapping keys and the node tree is -// re-encoded at the index's authored indent. That also makes the job -// idempotent: a run that computes the same values leaves the file untouched -// and opens no PR. -func writeVariants(path string, data []byte, entries []gallery.GalleryModel) error { - var doc yaml.Node - if err := yaml.Unmarshal(data, &doc); err != nil { - return fmt.Errorf("re-parse for node rewrite: %w", err) - } - if doc.Kind != yaml.DocumentNode || len(doc.Content) == 0 { - return fmt.Errorf("unexpected document shape") - } - root := doc.Content[0] - if root.Kind != yaml.SequenceNode { - return fmt.Errorf("index root is not a sequence") - } - if len(root.Content) != len(entries) { - return fmt.Errorf("entry count drift: %d nodes vs %d entries", len(root.Content), len(entries)) - } - - changed := false - for i, entryNode := range root.Content { - if !entries[i].HasVariants() || entryNode.Kind != yaml.MappingNode { - continue - } - - variantsNode := mappingValue(entryNode, "variants") - if variantsNode == nil || variantsNode.Kind != yaml.SequenceNode { - return fmt.Errorf("entry %q has variants but no variants sequence in the document", entries[i].Name) - } - if len(variantsNode.Content) != len(entries[i].Variants) { - return fmt.Errorf("entry %q variant count drift: %d nodes vs %d parsed", - entries[i].Name, len(variantsNode.Content), len(entries[i].Variants)) - } - - for j, variantNode := range variantsNode.Content { - if variantNode.Kind != yaml.MappingNode { - return fmt.Errorf("entry %q variant %d is not a mapping", entries[i].Name, j) - } - c := entries[i].Variants[j] - for _, kv := range []struct{ key, value string }{ - {"backend", c.Backend}, - {"quantization", c.Quantization}, - {"inferred_min_memory", c.InferredMinMemory}, - } { - if setMappingValue(variantNode, kv.key, kv.value) { - changed = true - } - } - } - } - - // Nothing to rewrite means nothing to encode: leave the file, and its - // mtime, alone. - if !changed { - return nil - } - - // yaml.Marshal encodes at yaml.v3's default 4-space indent, which reflows - // every nested block in the file (~6000 lines against the real index) and - // drowns the handful of rewritten keys. The index is authored at 2, so - // encode at 2 and the diff stays limited to what actually changed. - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode(&doc); err != nil { - return fmt.Errorf("encode: %w", err) - } - if err := enc.Close(); err != nil { - return fmt.Errorf("flush encoder: %w", err) - } - - // The encoder does not emit a document start marker, so restore the one the - // file was authored with rather than deleting it on every run. - out := buf.Bytes() - if header := documentHeader(data); header != nil && !bytes.HasPrefix(out, header) { - out = append(header, out...) - } - - // Preserve the file's existing mode instead of forcing 0644. - info, err := os.Stat(path) - if err != nil { - return fmt.Errorf("stat: %w", err) - } - return os.WriteFile(path, out, info.Mode().Perm()) -} - -// documentHeader returns the leading document start marker line, or nil when the -// file was authored without one. -func documentHeader(data []byte) []byte { - for _, marker := range [][]byte{[]byte("---\n"), []byte("---\r\n")} { - if bytes.HasPrefix(data, marker) { - return marker - } - } - return nil -} - -// mappingValue returns the value node for key, or nil when absent. -func mappingValue(mapping *yaml.Node, key string) *yaml.Node { - for i := 0; i+1 < len(mapping.Content); i += 2 { - if mapping.Content[i].Value == key { - return mapping.Content[i+1] - } - } - return nil -} - -// setMappingValue sets key to value, appending the key when absent and dropping -// it when value is empty so a field that stops being derivable does not leave a -// stale number behind. It reports whether the document actually changed. -func setMappingValue(mapping *yaml.Node, key, value string) bool { - for i := 0; i+1 < len(mapping.Content); i += 2 { - if mapping.Content[i].Value != key { - continue - } - if value == "" { - mapping.Content = append(mapping.Content[:i], mapping.Content[i+2:]...) - return true - } - if mapping.Content[i+1].Value == value { - return false - } - mapping.Content[i+1] = scalarNode(value) - return true - } - if value == "" { - return false - } - mapping.Content = append(mapping.Content, scalarNode(key), scalarNode(value)) - return true -} - -func scalarNode(value string) *yaml.Node { - return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value} -} - -// backendOf mirrors the priority used by resolveBackend in -// core/gallery/backend_resolve.go: an entry's overrides win over whatever the -// referenced config file declares. -func backendOf(m gallery.GalleryModel, resolved gallery.ModelConfig) string { - if b, ok := m.Overrides["backend"].(string); ok && b != "" { - return b - } - var cfg struct { - Backend string `yaml:"backend"` - } - if err := yaml.Unmarshal([]byte(resolved.ConfigFile), &cfg); err == nil { - return cfg.Backend - } - return "" -} - -// quantizationOf reports the quantization declared by a referenced entry's -// overrides, empty when it declares none. -func quantizationOf(m gallery.GalleryModel) string { - if q, ok := m.Overrides["quantization"].(string); ok { - return q - } - return "" -} diff --git a/.github/workflows/gallery-denormalize.yaml b/.github/workflows/gallery-denormalize.yaml deleted file mode 100644 index e58a83e1f..000000000 --- a/.github/workflows/gallery-denormalize.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Denormalize gallery meta entries -on: - schedule: - - cron: 0 22 * * * - workflow_dispatch: -jobs: - denormalize: - if: github.repository == 'mudler/LocalAI' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - # The program exits 1 when any candidate could not be estimated, but it - # still writes everything it did compute. Aborting here would let one - # unreachable candidate block the refresh of every other one, night after - # night, so record the status and let the PR open with the partial result. - - name: Denormalize meta candidates - id: denormalize - run: | - set +e - go run ./.github/ci/gallery_denormalize.go gallery/index.yaml - status=$? - set -e - echo "status=$status" >> "$GITHUB_OUTPUT" - if [ "$status" -ne 0 ]; then - echo "partial=yes" >> "$GITHUB_OUTPUT" - echo "note=**This run was partial.** At least one candidate could not be estimated, so some fields may be missing or stale. See the workflow log." >> "$GITHUB_OUTPUT" - else - echo "partial=no" >> "$GITHUB_OUTPUT" - echo "note=All candidates were processed successfully." >> "$GITHUB_OUTPUT" - fi - - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 - with: - token: ${{ secrets.UPDATE_BOT_TOKEN }} - push-to-fork: ci-forks/LocalAI - commit-message: ':arrow_up: Denormalize meta model candidates' - title: 'chore(model-gallery): :arrow_up: refresh meta candidate metadata' - branch: "update/gallery-denormalize" - body: | - Refreshes the denormalized `backend`, `quantization` and - `inferred_min_vram` fields on meta model entries. - - Authored `min_vram` values are never modified. - - ${{ steps.denormalize.outputs.note }} - signoff: true - # Surface the failure only after the PR exists, so the problem stays - # visible without discarding the work that did succeed. - - name: Report estimation failures - if: steps.denormalize.outputs.partial == 'yes' - run: | - echo "gallery_denormalize exited ${{ steps.denormalize.outputs.status }}: one or more candidates need a hand-authored min_vram." - exit 1 diff --git a/core/gallery/models.go b/core/gallery/models.go index d8bf109c1..1792d415d 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -9,6 +9,7 @@ import ( "path/filepath" "slices" "strings" + "time" "dario.cat/mergo" lconfig "github.com/mudler/LocalAI/core/config" @@ -17,6 +18,7 @@ import ( "github.com/mudler/LocalAI/pkg/modelartifacts" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" + "github.com/mudler/LocalAI/pkg/vram" "github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/xlog" @@ -88,7 +90,7 @@ type PromptTemplate struct { // The base is appended rather than special-cased downstream so there is exactly // one selection path, and so an operator can pin the entry's own name to // decline an upgrade their hardware would otherwise take. -func variantOptions(models []*GalleryModel, entry *GalleryModel) ([]VariantOption, error) { +func variantOptions(models []*GalleryModel, entry *GalleryModel, env ResolveEnv) ([]VariantOption, error) { options := make([]VariantOption, 0, len(entry.Variants)+1) for _, v := range entry.Variants { // Every referenced entry is looked up here rather than lazily after @@ -101,16 +103,73 @@ func variantOptions(models []*GalleryModel, entry *GalleryModel) ([]VariantOptio if target.HasVariants() { return nil, fmt.Errorf("model %q references variant %q which declares variants of its own; resolution is a single pass, so those would be silently ignored", entry.Name, v.Model) } - options = append(options, VariantOption{Variant: v, Backend: target.Backend}) + + option := VariantOption{Variant: v, Backend: target.Backend} + // An authored figure short circuits the probe: the author already + // answered the question the probe would spend a round trip asking. + if v.MinMemory == "" && env.ProbeMemory != nil { + option.ProbedMemory = env.ProbeMemory(target) + } + options = append(options, option) } + // The base is never filtered nor ranked, so its size would change nothing + // and probing it would spend a round trip on a discarded answer. return append(options, VariantOption{ - Variant: Variant{Model: entry.Name, MinMemory: entry.MinMemory}, + Variant: Variant{Model: entry.Name}, Backend: entry.Backend, IsBase: true, }), nil } +// probeContextLength is the context size the footprint estimate is taken at. +// Selection compares whole models against whole hosts, so this only has to be a +// consistent, realistic default rather than the context a user will eventually +// configure. +const probeContextLength = 8192 + +// probeTimeout bounds a single entry's probe. An entry with several variants +// probes each of them, so an unreachable host has to give up quickly: an +// unknown size only costs a variant its ranking, whereas a stalled probe costs +// the user the whole install. +const probeTimeout = 5 * time.Second + +// probeEntryMemory measures what a gallery entry will occupy, without +// downloading it. +// +// pkg/vram does the work and caches its results across calls: it range-fetches +// the GGUF header for a real estimate, falls back to an HTTP HEAD for the +// content length, and finally to any declared size:. A HuggingFace repo listing +// is deliberately NOT used as a further fallback, unlike the gallery UI's +// estimator: a quantization entry's urls routinely point at the base model's +// repo, and summing every weight file there would overstate one variant badly +// enough to wrongly filter it out. +// +// Zero means the size could not be determined. Callers must read that as +// unknown rather than as zero, so an unreachable network downgrades a variant's +// ranking instead of failing the install. +func probeEntryMemory(ctx context.Context, entry *GalleryModel) uint64 { + input := vram.ModelEstimateInput{Size: entry.Size} + for _, f := range entry.AdditionalFiles { + if vram.IsWeightFile(f.URI) { + input.Files = append(input.Files, vram.FileInput{URI: f.URI}) + } + } + if len(input.Files) == 0 && input.Size == "" { + return 0 + } + + ctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + + estimate, err := vram.EstimateModelMultiContext(ctx, input, []uint32{probeContextLength}) + if err != nil { + xlog.Debug("Could not probe a model variant's size; treating it as unknown", "model", entry.Name, "error", err) + return 0 + } + return estimate.VRAMForContext(probeContextLength) +} + // ResolveVariant picks the gallery entry to install for a host, from the // variants an entry declares plus the entry itself, and returns it renamed to // the entry's name and carrying the entry's presentation metadata. @@ -121,12 +180,13 @@ func variantOptions(models []*GalleryModel, entry *GalleryModel) ([]VariantOptio // from the entry the user asked for, so the installed model presents as that // model rather than as one of its variants. // -// The base entry always resolves. When even its own requirement falls short -// this installs it regardless, because the entry is a complete entry that every -// older LocalAI release installs unconditionally; refusing it here would make -// the gallery behave worse the newer the client is. +// The base entry always resolves, whatever the host has. It is a complete entry +// that every older LocalAI release installs unconditionally, so refusing it here +// would make the gallery behave worse the newer the client is. That is also why +// the base declares no memory requirement of its own: a floor that can only +// warn cannot change any outcome. func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Variant, error) { - options, err := variantOptions(models, entry) + options, err := variantOptions(models, entry, env) if err != nil { return nil, Variant{}, err } @@ -146,7 +206,7 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, // checks, but a silent bypass makes a later out-of-memory failure // impossible to trace back to the pin, so it is recorded loudly here. if pin != "" { - if need, known, verr := selected.Variant.EffectiveMinMemory(); verr == nil && known && env.AvailableMemory < need { + if need, known, verr := selected.EffectiveMemory(); verr == nil && known && env.AvailableMemory < need { xlog.Warn("Pinned model variant declares more memory than this system reports; installing anyway because the pin overrides hardware resolution", "model", entry.Name, "variant", selected.Variant.Model, "required_memory", need, "available_memory", env.AvailableMemory) } @@ -166,10 +226,9 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, resolved.Icon = entry.Icon resolved.License = entry.License // The resolved entry is a concrete install target, so it must not carry the - // selection fields any more; leaving them would let a second pass resolve - // the already-resolved entry all over again. + // variant list any more; leaving it would let a second pass resolve the + // already-resolved entry all over again. resolved.Variants = nil - resolved.MinMemory = "" // The struct copy above is shallow, so every reference-typed field still // aliases the gallery's own entries. The install path mutates Overrides in @@ -396,6 +455,9 @@ func InstallModelFromGallery( BackendCompatible: func(backend string) bool { return systemState.IsBackendCompatible(backend, "") }, + ProbeMemory: func(target *GalleryModel) uint64 { + return probeEntryMemory(ctx, target) + }, } resolved, variant, err := ResolveVariant(models, model, env, pin) diff --git a/core/gallery/models_types.go b/core/gallery/models_types.go index 2a48ca684..922ac1703 100644 --- a/core/gallery/models_types.go +++ b/core/gallery/models_types.go @@ -25,10 +25,6 @@ type GalleryModel struct { // stays a complete, installable entry and older LocalAI releases, which drop // this key, install it exactly as before. Variants []Variant `json:"variants,omitempty" yaml:"variants,omitempty"` - // MinMemory is this entry's own memory requirement (e.g. "2GiB"), in the - // same form as a variant's. It is advisory for the base entry: falling short - // only warns, because the base always installs. - MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"` } func (m *GalleryModel) GetInstalled() bool { diff --git a/core/gallery/resolve_variant.go b/core/gallery/resolve_variant.go index cdfceeae6..cb111bc61 100644 --- a/core/gallery/resolve_variant.go +++ b/core/gallery/resolve_variant.go @@ -30,6 +30,29 @@ type VariantOption struct { // filter and is the answer when nothing else survives, because the entry // must stay installable on every host and for every client. IsBase bool + // ProbedMemory is the footprint measured live from the referenced entry's + // weights, in bytes. It is only consulted when the variant declares no + // min_memory of its own, because a human who measured a real load knows + // more than a pre-download estimate does. + // + // Zero means the probe could not determine a size. That is an unknown, not + // a zero requirement: a probe that cannot reach the network must never be + // able to break an install. + ProbedMemory uint64 +} + +// EffectiveMemory returns this option's memory requirement in bytes and whether +// one is known at all: the authored figure when there is one, else the live +// probe result, else nothing. +func (o VariantOption) EffectiveMemory() (uint64, bool, error) { + size, known, err := o.Variant.AuthoredMinMemory() + if err != nil || known { + return size, known, err + } + if o.ProbedMemory > 0 { + return o.ProbedMemory, true, nil + } + return 0, false, nil } // ResolveEnv describes the host a variant is selected for. @@ -45,6 +68,18 @@ type ResolveEnv struct { // A nil func treats every backend as runnable, the right default for a // caller with no view of the hardware. BackendCompatible func(backend string) bool + // ProbeMemory measures how much memory a referenced gallery entry needs, + // without downloading it. It is consulted only for variants that declare no + // min_memory, and a zero result means "could not tell", never "needs + // nothing". + // + // It is a func field rather than a live network handle so specs can pin an + // exact size, or an exact failure, without reaching the internet. A nil func + // leaves every unauthored variant unknown, which selection already handles. + // + // SelectVariant never calls this: the install layer resolves every size into + // VariantOption.ProbedMemory first, so the selector stays pure. + ProbeMemory func(entry *GalleryModel) uint64 } func (e ResolveEnv) backendRuns(backend string) bool { @@ -75,8 +110,8 @@ type VariantSelection struct { // the hardware gate, derived from the backend name. // 3. Variants whose known memory requirement exceeds what the host has are // dropped. A variant with an UNKNOWN requirement survives, because nothing -// proves it does not fit and refusing on a missing estimate would punish -// the entries the denormalizer has not reached yet. +// proves it does not fit and refusing on a size the probe could not read +// would let a network hiccup silently downgrade what gets installed. // 4. The largest survivor wins. A bigger footprint is a higher quality // quantization of the same model, so among things that fit, more is better. // Unknown requirements rank last, so a proven fit always beats a guess. @@ -114,7 +149,7 @@ func SelectVariant(options []VariantOption, env ResolveEnv, pin string) (Variant continue } - memory, known, err := o.Variant.EffectiveMinMemory() + memory, known, err := o.EffectiveMemory() if err != nil { return VariantSelection{}, err } diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index 190fe3c1e..49fcb3468 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -7,34 +7,49 @@ import ( "github.com/mudler/LocalAI/core/gallery" ) -var _ = Describe("Variant.EffectiveMinMemory", func() { - It("reports no requirement when neither authored nor inferred", func() { - v := gallery.Variant{Model: "x"} - size, known, err := v.EffectiveMinMemory() +var _ = Describe("VariantOption.EffectiveMemory", func() { + It("reports no requirement when nothing is authored and nothing was probed", func() { + o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}} + size, known, err := o.EffectiveMemory() Expect(err).ToNot(HaveOccurred()) Expect(known).To(BeFalse()) Expect(size).To(Equal(uint64(0))) }) - It("uses the inferred value when nothing is authored", func() { - v := gallery.Variant{Model: "x", InferredMinMemory: "6GiB"} - size, known, err := v.EffectiveMinMemory() + It("uses the probed size when nothing is authored", func() { + o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}, ProbedMemory: 6 * 1024 * 1024 * 1024} + size, known, err := o.EffectiveMemory() Expect(err).ToNot(HaveOccurred()) Expect(known).To(BeTrue()) Expect(size).To(Equal(uint64(6 * 1024 * 1024 * 1024))) }) - It("lets an authored value win over an inferred one", func() { - v := gallery.Variant{Model: "x", MinMemory: "20GiB", InferredMinMemory: "6GiB"} - size, known, err := v.EffectiveMinMemory() + It("lets an authored figure win over a probed one", func() { + // A human who measured a real load knows more than a pre-download + // estimate does, which is the entire reason min_memory survives. + o := gallery.VariantOption{ + Variant: gallery.Variant{Model: "x", MinMemory: "20GiB"}, + ProbedMemory: 6 * 1024 * 1024 * 1024, + } + size, known, err := o.EffectiveMemory() Expect(err).ToNot(HaveOccurred()) Expect(known).To(BeTrue()) Expect(size).To(Equal(uint64(20 * 1024 * 1024 * 1024))) }) - It("errors on an unparseable figure rather than silently treating it as absent", func() { - v := gallery.Variant{Model: "x", MinMemory: "twenty gigs"} - _, _, err := v.EffectiveMinMemory() + It("treats a failed probe as unknown rather than as a zero requirement", func() { + // A probe that could not reach the network reports 0. Reading that as + // "needs nothing" would make an unreachable host look like the perfect + // fit and hand the user the largest download on offer. + o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}, ProbedMemory: 0} + _, known, err := o.EffectiveMemory() + Expect(err).ToNot(HaveOccurred()) + Expect(known).To(BeFalse()) + }) + + It("errors on an unparseable authored figure rather than silently ignoring it", func() { + o := gallery.VariantOption{Variant: gallery.Variant{Model: "x", MinMemory: "twenty gigs"}} + _, _, err := o.EffectiveMemory() Expect(err).To(HaveOccurred()) }) }) @@ -49,9 +64,13 @@ var _ = Describe("SelectVariant", func() { } } - base := func(model, minMemory string) gallery.VariantOption { - o := option(model, "llama-cpp", minMemory) + // The base entry declares no memory requirement of its own, so a base + // option's size can only ever come from a probe. It is exempt from every + // filter regardless, which the fallback specs below pin down. + base := func(model string, probed uint64) gallery.VariantOption { + o := option(model, "llama-cpp", "") o.IsBase = true + o.ProbedMemory = probed return o } @@ -68,7 +87,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-mlx-8bit", "mlx", "24GiB"), option("m-gguf-q8", "llama-cpp", "12GiB"), - base("m-gguf-q4", "6GiB"), + base("m-gguf-q4", gib(6)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ @@ -85,7 +104,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-mlx-8bit", "mlx", "24GiB"), option("m-gguf-q8", "llama-cpp", "12GiB"), - base("m-gguf-q4", "6GiB"), + base("m-gguf-q4", gib(6)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ @@ -97,7 +116,7 @@ var _ = Describe("SelectVariant", func() { }) It("treats every backend as runnable when the host cannot be inspected", func() { - options := []gallery.VariantOption{option("m-mlx-8bit", "mlx", "24GiB"), base("m-gguf-q4", "6GiB")} + options := []gallery.VariantOption{option("m-mlx-8bit", "mlx", "24GiB"), base("m-gguf-q4", gib(6))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(80)}, "") Expect(err).ToNot(HaveOccurred()) @@ -113,7 +132,7 @@ var _ = Describe("SelectVariant", func() { option("m-q4", "llama-cpp", "6GiB"), option("m-q8", "llama-cpp", "12GiB"), option("m-f16", "llama-cpp", "24GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") @@ -129,7 +148,7 @@ var _ = Describe("SelectVariant", func() { option("m-f16", "llama-cpp", "24GiB"), option("m-q8", "llama-cpp", "12GiB"), option("m-q4", "llama-cpp", "6GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") @@ -144,7 +163,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-unknown", "llama-cpp", ""), option("m-q8", "llama-cpp", "12GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") @@ -155,7 +174,7 @@ var _ = Describe("SelectVariant", func() { It("keeps a variant of unknown size rather than dropping it", func() { options := []gallery.VariantOption{ option("m-unknown", "llama-cpp", ""), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") @@ -164,8 +183,30 @@ var _ = Describe("SelectVariant", func() { Expect(selection.FellBackToBase).To(BeFalse()) }) + It("ranks and filters a probed size exactly as an authored one", func() { + // Nothing here declares min_memory, so every figure came from the + // live probe. The probe is the primary source now and authoring the + // exception, so it has to drive both the filter and the ranking. + probed := func(model string, size uint64) gallery.VariantOption { + o := option(model, "llama-cpp", "") + o.ProbedMemory = size + return o + } + options := []gallery.VariantOption{ + probed("m-q4", gib(6)), + probed("m-f16", gib(24)), + probed("m-q8", gib(12)), + base("m-base", gib(2)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-q8")) + Expect(selection.Reasons).To(ContainElement(ContainSubstring("m-f16"))) + }) + It("admits a variant needing exactly the memory available", func() { - options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(12)}, "") Expect(err).ToNot(HaveOccurred()) @@ -178,7 +219,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-q8", "llama-cpp", "12GiB"), option("m-f16", "llama-cpp", "24GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") @@ -189,10 +230,11 @@ var _ = Describe("SelectVariant", func() { Expect(selection.Reasons).To(HaveLen(2)) }) - It("selects the base even when the base's own requirement is unmet", func() { - // There is nothing below the base, so refusing here would make an - // entry every older client installs fine uninstallable on newer ones. - options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", "2GiB")} + It("selects the base even when the base does not fit either", func() { + // The base is exempt from the memory filter, not merely favoured by + // it: there is nothing below it, so refusing here would make an entry + // every older client installs fine uninstallable on newer ones. + options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: 0}, "") Expect(err).ToNot(HaveOccurred()) @@ -203,7 +245,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-mlx", "mlx", "8GiB"), option("m-f16", "llama-cpp", "24GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ @@ -227,7 +269,7 @@ var _ = Describe("SelectVariant", func() { It("honors a pin the hardware would never have chosen", func() { options := []gallery.VariantOption{ option("m-mlx", "mlx", "64GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ @@ -239,7 +281,7 @@ var _ = Describe("SelectVariant", func() { }) It("honors a pin naming the base, declining an upgrade that fits", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-base") Expect(err).ToNot(HaveOccurred()) @@ -247,7 +289,7 @@ var _ = Describe("SelectVariant", func() { }) It("matches a pin case-insensitively", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "M-F16") Expect(err).ToNot(HaveOccurred()) @@ -255,7 +297,7 @@ var _ = Describe("SelectVariant", func() { }) It("fails loudly when the pin names nothing in the list", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-gone") Expect(err).To(MatchError(gallery.ErrPinNotFound)) @@ -264,7 +306,7 @@ var _ = Describe("SelectVariant", func() { }) It("propagates an unparseable figure instead of treating it as unconstrained", func() { - options := []gallery.VariantOption{option("bad", "llama-cpp", "lots"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("bad", "llama-cpp", "lots"), base("m-base", gib(2))} _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(8)}, "") Expect(err).To(HaveOccurred()) diff --git a/core/gallery/variant.go b/core/gallery/variant.go index 0821f440a..35ca8bfb7 100644 --- a/core/gallery/variant.go +++ b/core/gallery/variant.go @@ -16,41 +16,29 @@ type Variant struct { // Model is the name of a gallery entry that declares no variants of its own. Model string `json:"model" yaml:"model"` // MinMemory is the authored memory requirement (e.g. "20GiB"). It exists to - // override an inferred estimate that is known to be wrong; most variants - // should not need it. + // override the size the installer probes live from the model's weights when + // that figure is known to be wrong; most variants should not need it. // // One number covers both VRAM and system RAM. A model's footprint is // roughly the same wherever it lives, and the selector compares this against // whichever of the two this host will actually use, so splitting it in two // would only invite the pair to disagree. MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"` - - // The fields below are denormalized by the nightly job for display and - // lint. They are never authored by hand and never affect what gets - // installed, because installation reads the referenced entry live. - Backend string `json:"backend,omitempty" yaml:"backend,omitempty"` - Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"` - InferredMinMemory string `json:"inferred_min_memory,omitempty" yaml:"inferred_min_memory,omitempty"` } -// EffectiveMinMemory returns this variant's memory requirement in bytes and -// whether one is known at all. An authored MinMemory wins over an inferred one, -// because a human who measured a real load knows more than a pre-download -// estimate does. +// AuthoredMinMemory returns the hand-written memory requirement in bytes and +// whether one was written at all. // -// An unknown requirement is not a zero requirement. Callers must not read the -// returned 0 as "fits anywhere"; it means nothing is known either way. -func (v Variant) EffectiveMinMemory() (uint64, bool, error) { - raw := v.MinMemory - if raw == "" { - raw = v.InferredMinMemory - } - if raw == "" { +// An absent requirement is not a zero requirement. Callers must not read the +// returned 0 as "fits anywhere"; it means the author said nothing, and the +// figure has to come from a live probe instead. +func (v Variant) AuthoredMinMemory() (uint64, bool, error) { + if v.MinMemory == "" { return 0, false, nil } - size, err := vram.ParseSizeString(raw) + size, err := vram.ParseSizeString(v.MinMemory) if err != nil { - return 0, false, fmt.Errorf("variant %q has an unparseable min_memory %q: %w", v.Model, raw, err) + return 0, false, fmt.Errorf("variant %q has an unparseable min_memory %q: %w", v.Model, v.MinMemory, err) } return size, true, nil } diff --git a/core/gallery/variants_install_test.go b/core/gallery/variants_install_test.go index 1de4e5ace..424bbfd81 100644 --- a/core/gallery/variants_install_test.go +++ b/core/gallery/variants_install_test.go @@ -28,12 +28,11 @@ var _ = Describe("GalleryModel variant declarations", func() { Expect(m.HasVariants()).To(BeTrue()) }) - It("parses an entry's own memory figure and its variant list", func() { + It("parses a variant list", func() { var m gallery.GalleryModel err := yaml.Unmarshal([]byte(` name: qwen3.6-27b url: "github:example/repo/qwen3.6-27b.yaml@master" -min_memory: 4GiB variants: - model: qwen3.6-27b-mlx-8bit - model: qwen3.6-27b-gguf-q8 @@ -42,7 +41,6 @@ variants: Expect(err).ToNot(HaveOccurred()) Expect(m.Name).To(Equal("qwen3.6-27b")) Expect(m.URL).To(Equal("github:example/repo/qwen3.6-27b.yaml@master")) - Expect(m.MinMemory).To(Equal("4GiB")) Expect(m.HasVariants()).To(BeTrue()) Expect(m.Variants).To(HaveLen(2)) // The first variant is nothing but a name, which is the shape authoring @@ -81,7 +79,6 @@ var _ = Describe("ResolveVariant", func() { base = newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "Qwen3 8B Q4", "qwen.png") base.Backend = "llama-cpp" base.Tags = []string{"llm"} - base.MinMemory = "6GiB" base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} models = []*gallery.GalleryModel{upgrade, base} }) @@ -118,20 +115,96 @@ var _ = Describe("ResolveVariant", func() { Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) - It("installs the entry even when the host misses the entry's own requirement", func() { - resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(1)), "") + It("installs the entry on a host with no memory budget at all", func() { + // The base carries no floor of its own precisely so this can never + // refuse; an unreadable host reports 0 and must still get the entry. + resolved, variant, err := gallery.ResolveVariant(models, base, env(0), "") Expect(err).ToNot(HaveOccurred()) Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) - It("strips the selection fields from the resolved entry", func() { - // A resolved entry is a concrete install target. Leaving the fields on - // it would let a second selection pass fire on an already-resolved entry. + It("strips the variant list from the resolved entry", func() { + // A resolved entry is a concrete install target. Leaving the list on it + // would let a second selection pass fire on an already-resolved entry. resolved, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "") Expect(err).ToNot(HaveOccurred()) Expect(resolved.HasVariants()).To(BeFalse()) - Expect(resolved.MinMemory).To(BeEmpty()) + }) + + Describe("the live size probe", func() { + // The probe is injected rather than performed, so these specs pin an + // exact size, or an exact failure, without reaching the network. + probing := func(memory uint64, sizes map[string]uint64) gallery.ResolveEnv { + e := env(memory) + e.ProbeMemory = func(target *gallery.GalleryModel) uint64 { return sizes[target.Name] } + return e + } + + BeforeEach(func() { + // No authored figure anywhere, so every size below comes from the + // probe. This is the shape authoring is meant to reach for. + base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq"}} + }) + + It("selects a variant the probe shows to fit", func() { + resolved, variant, err := gallery.ResolveVariant(models, base, + probing(gib(24), map[string]uint64{"qwen3-8b-vllm-awq": gib(20)}), "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("rejects a variant the probe shows to be too large", func() { + // Same variant, same host, only the probed size differs, so nothing + // but the probe can account for the different answer. + resolved, variant, err := gallery.ResolveVariant(models, base, + probing(gib(24), map[string]uint64{"qwen3-8b-vllm-awq": gib(80)}), "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("completes the install when the probe cannot determine a size", func() { + // A failed probe reports 0. That is an unknown, so the variant is + // not filtered out, and above all the resolution does not fail: a + // network hiccup must never be able to break an install. + resolved, variant, err := gallery.ResolveVariant(models, base, + probing(gib(24), map[string]uint64{}), "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("still installs the base when every probe fails and nothing else survives", func() { + // The one variant this host could otherwise take is ruled out by its + // backend and no probe answers, so selection has to terminate on the + // base rather than on an error. + e := probing(gib(24), map[string]uint64{}) + e.BackendCompatible = func(backend string) bool { return backend != "vllm" } + + resolved, variant, err := gallery.ResolveVariant(models, base, e, "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("does not probe a variant that declares its own min_memory", func() { + // Probing costs a network round trip, so an authored figure has to + // suppress it outright rather than merely outrank it. + base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} + probed := []string{} + e := env(gib(24)) + e.ProbeMemory = func(target *gallery.GalleryModel) uint64 { + probed = append(probed, target.Name) + return gib(80) + } + + _, variant, err := gallery.ResolveVariant(models, base, e, "") + Expect(err).ToNot(HaveOccurred()) + Expect(probed).To(BeEmpty()) + Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) + }) }) It("presents the entry's metadata, not the variant's", func() { @@ -360,6 +433,22 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { Expect(os.IsNotExist(err)).To(BeTrue(), "the variant must not be installed under its own name") }) + It("completes the install when a variant's size cannot be determined", func() { + // This runs the REAL probe against an entry that declares no size and no + // weight files, which is exactly the shape a probe cannot answer for. + // It must yield an unknown, not an error: the variant survives the + // filter and the install completes, on the variant or on the base, but + // never on a failure. Nothing here touches the network. + newGallery( + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(BeElementOf("base-backend", "upgrade-backend")) + }) + It("installs the largest fitting variant, not the first authored", func() { // Authored smallest-first with all three within reach, so first-match // would take the small one. diff --git a/core/gallery/variants_lint_test.go b/core/gallery/variants_lint_test.go index 4d21d1741..a60de7783 100644 --- a/core/gallery/variants_lint_test.go +++ b/core/gallery/variants_lint_test.go @@ -75,12 +75,12 @@ func checkVariantReferences(entries []gallery.GalleryModel) []variantViolation { return violations } -// checkVariantMemory verifies every declared memory figure parses, on the -// variants and on the entry itself. +// checkVariantMemory verifies every authored memory figure parses. // -// Absence is fine and deliberately not flagged: an unknown requirement is a -// legitimate state that selection handles by ranking the variant last. An -// UNPARSEABLE one is different, because it makes selection fail outright for +// Absence is fine and deliberately not flagged: a variant that declares nothing +// gets its size from a live probe, and even a probe that fails leaves a +// legitimate unknown that selection handles by ranking the variant last. An +// UNPARSEABLE figure is different, because it makes selection fail outright for // the whole entry rather than degrade. func checkVariantMemory(entries []gallery.GalleryModel) []variantViolation { var violations []variantViolation @@ -88,12 +88,8 @@ func checkVariantMemory(entries []gallery.GalleryModel) []variantViolation { if !e.HasVariants() { continue } - base := gallery.Variant{Model: e.Name, MinMemory: e.MinMemory} - if _, _, err := base.EffectiveMinMemory(); err != nil { - violations = append(violations, variantViolation{Entry: e.Name, Detail: "has a bad min_memory: " + err.Error()}) - } for _, v := range e.Variants { - if _, _, err := v.EffectiveMinMemory(); err != nil { + if _, _, err := v.AuthoredMinMemory(); err != nil { violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "has a bad min_memory: " + err.Error()}) } } @@ -123,8 +119,8 @@ func plainEntry(name, url string) gallery.GalleryModel { return e } -func entryWithVariants(name, url, minMemory string, variants ...gallery.Variant) gallery.GalleryModel { - e := gallery.GalleryModel{Variants: variants, MinMemory: minMemory} +func entryWithVariants(name, url string, variants ...gallery.Variant) gallery.GalleryModel { + e := gallery.GalleryModel{Variants: variants} e.Name = name e.URL = url return e @@ -144,13 +140,13 @@ var _ = Describe("gallery variant lint helpers", func() { // well-meaning alignment of the two would make every helper skip every // entry and pass silently. Assert the distinction directly. It("treats an entry with variants as such even though it has a url", func() { - Expect(entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "big"}).HasVariants()).To(BeTrue()) + Expect(entryWithVariants("base", "u://base", gallery.Variant{Model: "big"}).HasVariants()).To(BeTrue()) Expect(plainEntry("big", "u://big").HasVariants()).To(BeFalse()) }) It("passes every invariant on a valid entry", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "4GiB", + entryWithVariants("base", "u://base", gallery.Variant{Model: "big", MinMemory: "24GiB"}, gallery.Variant{Model: "mid", MinMemory: "12GiB"}, gallery.Variant{Model: "metal-big", MinMemory: "32GiB"}, @@ -167,10 +163,10 @@ var _ = Describe("gallery variant lint helpers", func() { }) It("passes every invariant on an entry that declares no memory at all", func() { - // Authoring is meant to be nothing but a list of names, so an entry and - // its variants carrying no figures must be entirely valid. + // Authoring is meant to be nothing but a list of names, so an entry + // whose variants carry no figures must be entirely valid. entries := variantFixture( - entryWithVariants("base", "u://base", "", gallery.Variant{Model: "big"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "big"}), plainEntry("big", "u://big"), ) @@ -181,7 +177,7 @@ var _ = Describe("gallery variant lint helpers", func() { Describe("checkVariantReferences", func() { It("flags a variant naming an entry that does not exist", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "ghost"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "ghost"}), plainEntry("a", "u://a"), ) @@ -193,8 +189,8 @@ var _ = Describe("gallery variant lint helpers", func() { It("flags a variant naming an entry that declares variants itself", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "nested"}), - entryWithVariants("nested", "u://nested", "4GiB", gallery.Variant{Model: "a"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "nested"}), + entryWithVariants("nested", "u://nested", gallery.Variant{Model: "a"}), plainEntry("a", "u://a"), ) @@ -206,7 +202,7 @@ var _ = Describe("gallery variant lint helpers", func() { It("reports every breach in one pass rather than stopping at the first", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "2GiB", + entryWithVariants("base", "u://base", gallery.Variant{Model: "ghost"}, gallery.Variant{Model: "phantom"}, ), @@ -219,7 +215,7 @@ var _ = Describe("gallery variant lint helpers", func() { Describe("checkVariantMemory", func() { It("flags a variant whose memory figure cannot be parsed", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "a", MinMemory: "lots"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "a", MinMemory: "lots"}), plainEntry("a", "u://a"), ) @@ -229,30 +225,16 @@ var _ = Describe("gallery variant lint helpers", func() { Expect(violations[0].Detail).To(ContainSubstring("bad min_memory")) }) - It("flags an entry whose own memory figure cannot be parsed", func() { + It("accepts a variant that declares no figure at all", func() { + // The overwhelmingly common shape: a bare name, whose size the + // installer probes live. Flagging it would make the rule reject the + // authoring style the design exists to enable. entries := variantFixture( - entryWithVariants("base", "u://base", "eight gigs", gallery.Variant{Model: "a"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "a"}), plainEntry("a", "u://a"), ) - violations := checkVariantMemory(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Entry).To(Equal("base")) - Expect(violations[0].Variant).To(BeEmpty()) - Expect(violations[0].Detail).To(ContainSubstring("bad min_memory")) - }) - - It("flags an unparseable denormalized figure just as an authored one", func() { - // The nightly job writes inferred_min_memory, so a bug there breaks - // selection exactly as a bad hand-authored value does. - entries := variantFixture( - entryWithVariants("base", "u://base", "", gallery.Variant{Model: "a", InferredMinMemory: "8 gigabytes"}), - plainEntry("a", "u://a"), - ) - - violations := checkVariantMemory(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Variant).To(Equal("a")) + Expect(checkVariantMemory(entries)).To(BeEmpty()) }) }) }) diff --git a/core/schema/gallery-model.schema.json b/core/schema/gallery-model.schema.json index 998da2ec3..0cf0184d1 100644 --- a/core/schema/gallery-model.schema.json +++ b/core/schema/gallery-model.schema.json @@ -56,10 +56,6 @@ "additionalProperties": false } }, - "min_memory": { - "type": "string", - "description": "This entry's own memory requirement, for example 2GiB. One figure covers both VRAM and system RAM; the installer compares it against whichever the host will use. Advisory for the entry itself: falling short only warns." - }, "variants": { "type": "array", "description": "Other builds of the same model (other backends, other quantizations) the installer may pick instead of this entry's own payload. Order carries no meaning: the installer drops what this host cannot run or does not have the memory for, then takes the largest of what is left, installing it under this entry's own name. If nothing is left this entry installs itself. Clients that predate variants support drop this key and install the entry unchanged.", @@ -74,19 +70,7 @@ }, "min_memory": { "type": "string", - "description": "Authored memory requirement, for example 20GiB. Optional, and only needed to override an inferred estimate that is wrong. Authoritative: inference never overwrites it." - }, - "backend": { - "type": "string", - "description": "Denormalized by the nightly job for display. Never authored by hand." - }, - "quantization": { - "type": "string", - "description": "Denormalized by the nightly job for display. Never authored by hand." - }, - "inferred_min_memory": { - "type": "string", - "description": "Denormalized by the nightly job as a fallback requirement. Never authored by hand; min_memory wins." + "description": "Authored memory requirement, for example 20GiB. Optional, and only needed to override the size the installer probes live from the model's weights when that figure is wrong. Authoritative: an authored figure suppresses the probe entirely." } }, "additionalProperties": false diff --git a/gallery/index.yaml b/gallery/index.yaml index c34b33fe1..a91089bfb 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -4297,13 +4297,11 @@ last_checked: "2026-04-30" # This entry installs the Q4_K_M build as-is, on any client and any host. # variants lists other builds of the same model the installer may pick - # instead when this host can run them and has the memory for them. Clients - # that predate variants support drop both keys and install this entry - # unchanged. - min_memory: 2GiB + # instead when this host can run them and has the memory for them, sizing + # each one live from its weights. Clients that predate variants support drop + # the key and install this entry unchanged. variants: - model: nanbeige4.1-3b-q8 - min_memory: 6GiB overrides: parameters: model: nanbeige4.1-3b-q4_k_m.gguf