mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-30 09:57:57 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(gallery): select model variants by hardware fit, not authored order

Browse Source 
Gallery entries could already carry a list of alternatives, but selection was
an authored, ordered, first-match policy: every candidate declared a
`capability` string and the VRAM floors had to descend in a hand-tuned order.
That pushed hardware knowledge onto whoever edits the gallery and made ordering
load-bearing, so a reordered list silently changed what users installed.

None of it was necessary. SystemState.IsBackendCompatible already derives
hardware support from a backend name alone: it knows MLX and metal are
Darwin-only, CUDA is NVIDIA-only, ROCm AMD-only, SYCL Intel-only. Selection can
read that instead of asking authors to restate it.

Authoring is now just a list of names:

    - name: qwen3.6-27b
      min_memory: 4GiB
      variants:
        - model: qwen3.6-27b-mlx-8bit
        - model: qwen3.6-27b-gguf-q8
          min_memory: 28GiB

and all the intelligence moved into the selector. Given a host it drops the
variants whose backend cannot run here, drops those whose known memory
requirement exceeds what the host has, and takes the LARGEST of what is left,
because a bigger footprint is a higher quality quantization of the same model.
A variant of unknown size is kept, since nothing proves it does not fit, but it
ranks last so a proven fit always beats a guess. An explicit pin still wins
outright, and if nothing survives the entry installs its own payload: the base
always installs, this never refuses.

Available memory is VRAM when a GPU was detected and system RAM otherwise, read
through xsysinfo so a cgroup limit is honored and a container gets its own
limit rather than the node's RAM.

Capability disappears entirely, from the types, the schema and the lint. VRAM
and RAM collapse into one `min_memory`, because a model's footprint is roughly
the same wherever it lives and one figure is compared against whichever applies.
The lint rules about ordering, the capability vocabulary and floor
relationships are deleted with the hazards they described; what remains is that
every variant names an entry that exists and does not itself declare variants,
plus that any memory figure actually parses.

Assisted-by: Claude:claude-opus-4-8
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2026-07-18 22:05:27 +00:00
parent 30b307293a
commit d7eb059f44
15 changed files with 1111 additions and 1046 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
.github/ci/gallery_denormalize.go vendored
View File

@@ -1,8 +1,8 @@
// Command gallery_denormalize fills the read-only denormalized fields on the
// candidates of gallery model entries: backend, quantization, and
// inferred_min_vram.
// variants of gallery model entries: backend, quantization, and
// inferred_min_memory.
//
// It never modifies an authored min_vram. Authored values are authoritative,
// It never modifies an authored min_memory. Authored values are authoritative,
// because a human who measured a real load knows more than a pre-download
// estimate does.
package main
@@ -51,20 +51,20 @@ func main() {
failures := 0
for i := range entries {
if !entries[i].HasCandidates() {
if !entries[i].HasVariants() {
continue
}
for j := range entries[i].Candidates {
c := &entries[i].Candidates[j]
for j := range entries[i].Variants {
c := &entries[i].Variants[j]
target, ok := byName[c.Model]
if !ok {
fmt.Fprintf(os.Stderr, "%s: candidate %q not found\n", entries[i].Name, c.Model)
fmt.Fprintf(os.Stderr, "%s: variant %q not found\n", entries[i].Name, c.Model)
failures++
continue
}
// The concrete entry's payload usually lives behind its url
// The referenced entry's payload usually lives behind its url
// rather than inline, so fetch it. Metadata.Backend is populated
// at load time by the running server, not by parsing the index,
// so it is always empty here and cannot be copied.
@@ -78,18 +78,17 @@ func main() {
c.Backend = backendOf(target, resolved)
c.Quantization = quantizationOf(target)
// Authored values win, and the final unconstrained candidate is
// deliberately floorless, so neither gets an inferred value.
// Clearing first matters: a candidate that gained an authored
// min_vram, or that became the last resort after a reorder, would
// otherwise keep a stale inferred floor that makes
// EffectiveMinVRAM report a constraint the entry no longer has.
if c.MinVRAM != "" || j == len(entries[i].Candidates)-1 {
c.InferredMinVRAM = ""
// An authored value wins outright, so it gets no inferred
// counterpart. Clearing first matters: a variant that gained an
// authored min_memory would otherwise keep a stale inferred figure
// that EffectiveMinMemory would go on reporting once the authored
// one is removed again.
if c.MinMemory != "" {
c.InferredMinMemory = ""
continue
}
// Weight files can come from either side: a concrete index entry
// Weight files can come from either side: a referenced index entry
// usually lists them itself (files:, i.e. AdditionalFiles) while
// the url it points at is only a base config, but some entries
// invert that. Install time downloads both, so estimate over both.
@@ -107,38 +106,38 @@ func main() {
}, []uint32{estimateContext})
if err != nil || estimate.VRAMForContext(estimateContext) == 0 {
fmt.Fprintf(os.Stderr,
"%s: could not estimate min_vram for %q (%v); author one by hand\n",
"%s: could not estimate min_memory for %q (%v); author one by hand\n",
entries[i].Name, c.Model, err)
failures++
continue
}
c.InferredMinVRAM = fmt.Sprintf("%dMiB", estimate.VRAMForContext(estimateContext)/(1024*1024))
c.InferredMinMemory = fmt.Sprintf("%dMiB", estimate.VRAMForContext(estimateContext)/(1024*1024))
}
}
if err := writeCandidates(path, data, entries); err != nil {
if err := writeVariants(path, data, entries); err != nil {
fmt.Fprintf(os.Stderr, "write %s: %v\n", path, err)
os.Exit(1)
}
if failures > 0 {
fmt.Fprintf(os.Stderr, "%d candidate(s) need a hand-authored min_vram\n", failures)
fmt.Fprintf(os.Stderr, "%d variant(s) need a hand-authored min_memory\n", failures)
os.Exit(1)
}
}
// writeCandidates writes back only the three derived keys on each candidate,
// writeVariants writes back only the three derived keys on each variant,
// leaving every other byte of the index alone.
//
// Round-tripping through []GalleryModel would reflow all 1200+ entries
// (quoting, key order, line wrapping), burying the handful of real changes in
// reformatting noise and making the nightly PR unreviewable. Even re-encoding
// just the candidates subtree would restyle authored fields such as min_vram,
// just the variants subtree would restyle authored fields such as min_memory,
// so the rewrite reaches down to individual mapping keys and the node tree is
// re-encoded at the index's authored indent. That also makes the job
// idempotent: a run that computes the same values leaves the file untouched
// and opens no PR.
func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) error {
func writeVariants(path string, data []byte, entries []gallery.GalleryModel) error {
var doc yaml.Node
if err := yaml.Unmarshal(data, &doc); err != nil {
return fmt.Errorf("re-parse for node rewrite: %w", err)
@@ -156,30 +155,30 @@ func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) e
changed := false
for i, entryNode := range root.Content {
if !entries[i].HasCandidates() || entryNode.Kind != yaml.MappingNode {
if !entries[i].HasVariants() || entryNode.Kind != yaml.MappingNode {
continue
}
candidatesNode := mappingValue(entryNode, "candidates")
if candidatesNode == nil || candidatesNode.Kind != yaml.SequenceNode {
return fmt.Errorf("entry %q has candidates but no candidates sequence in the document", entries[i].Name)
variantsNode := mappingValue(entryNode, "variants")
if variantsNode == nil || variantsNode.Kind != yaml.SequenceNode {
return fmt.Errorf("entry %q has variants but no variants sequence in the document", entries[i].Name)
}
if len(candidatesNode.Content) != len(entries[i].Candidates) {
return fmt.Errorf("entry %q candidate count drift: %d nodes vs %d parsed",
entries[i].Name, len(candidatesNode.Content), len(entries[i].Candidates))
if len(variantsNode.Content) != len(entries[i].Variants) {
return fmt.Errorf("entry %q variant count drift: %d nodes vs %d parsed",
entries[i].Name, len(variantsNode.Content), len(entries[i].Variants))
}
for j, candidateNode := range candidatesNode.Content {
if candidateNode.Kind != yaml.MappingNode {
return fmt.Errorf("entry %q candidate %d is not a mapping", entries[i].Name, j)
for j, variantNode := range variantsNode.Content {
if variantNode.Kind != yaml.MappingNode {
return fmt.Errorf("entry %q variant %d is not a mapping", entries[i].Name, j)
}
c := entries[i].Candidates[j]
c := entries[i].Variants[j]
for _, kv := range []struct{ key, value string }{
{"backend", c.Backend},
{"quantization", c.Quantization},
{"inferred_min_vram", c.InferredMinVRAM},
{"inferred_min_memory", c.InferredMinMemory},
} {
if setMappingValue(candidateNode, kv.key, kv.value) {
if setMappingValue(variantNode, kv.key, kv.value) {
changed = true
}
}
@@ -287,7 +286,7 @@ func backendOf(m gallery.GalleryModel, resolved gallery.ModelConfig) string {
return ""
}
// quantizationOf reports the quantization declared by a concrete entry's
// quantizationOf reports the quantization declared by a referenced entry's
// overrides, empty when it declares none.
func quantizationOf(m gallery.GalleryModel) string {
if q, ok := m.Overrides["quantization"].(string); ok {

49
core/gallery/candidate.go
View File

@@ -1,49 +0,0 @@
package gallery
import (
"fmt"
"github.com/mudler/LocalAI/pkg/vram"
)
// Candidate is one option in a gallery entry's ordered candidate list. It
// references an existing gallery entry by name and declares the conditions
// under which that entry is the right choice for the host.
type Candidate struct {
// Model is the name of a gallery entry that declares no candidates of its
// own, or the name of the declaring entry itself when it stands as its own
// last-resort candidate.
Model string `json:"model" yaml:"model"`
// Capability, when set, must equal the host's reported capability
// (e.g. "metal", "nvidia-cuda-12"). Empty matches any host.
Capability string `json:"capability,omitempty" yaml:"capability,omitempty"`
// MinVRAM is the authored VRAM floor (e.g. "20GiB"). Authoritative:
// inference never overwrites it.
MinVRAM string `json:"min_vram,omitempty" yaml:"min_vram,omitempty"`
// The fields below are denormalized by the nightly job for display and
// lint. They are never authored by hand and never affect what gets
// installed, because installation reads the referenced entry live.
Backend string `json:"backend,omitempty" yaml:"backend,omitempty"`
Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"`
InferredMinVRAM string `json:"inferred_min_vram,omitempty" yaml:"inferred_min_vram,omitempty"`
}
// EffectiveMinVRAM returns the VRAM floor in bytes for this candidate and
// whether a floor is declared at all. An authored MinVRAM wins over an
// inferred one. A candidate with no floor declares no requirement and always
// passes the VRAM check.
func (c Candidate) EffectiveMinVRAM() (uint64, bool, error) {
raw := c.MinVRAM
if raw == "" {
raw = c.InferredMinVRAM
}
if raw == "" {
return 0, false, nil
}
bytes, err := vram.ParseSizeString(raw)
if err != nil {
return 0, false, fmt.Errorf("candidate %q has an unparseable min_vram %q: %w", c.Model, raw, err)
}
return bytes, true, nil
}

544
core/gallery/candidates_lint_test.go
View File

@@ -1,544 +0,0 @@
package gallery_test
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gopkg.in/yaml.v3"
"github.com/mudler/LocalAI/core/gallery"
)
// knownCapabilities mirrors the values SystemState.DetectedCapability() can
// actually report, which is what the candidate resolver compares against with
// a case-sensitive exact match. A capability outside this set can never match,
// so a typo would silently make a candidate unreachable on every host.
//
// Note "cpu" is deliberately absent: it exists only as a fallback key inside
// SystemState.Capability(capMap) for meta backends, and is never a value
// getSystemCapabilities() returns. A CPU-only host reports "default".
var knownCapabilities = map[string]bool{
"default": true, "metal": true, "darwin-x86": true,
"nvidia": true, "nvidia-cuda-12": true, "nvidia-cuda-13": true,
"nvidia-l4t": true, "nvidia-l4t-cuda-12": true, "nvidia-l4t-cuda-13": true,
"intel": true, "amd": true, "vulkan": true,
}
// candidateViolation is one invariant breach found by a lint helper. Helpers
// return every breach they find instead of stopping at the first, so a single
// run names them all rather than forcing a fix-one-rerun-repeat cycle.
type candidateViolation struct {
Entry string
Candidate string
Detail string
}
func (v candidateViolation) String() string {
if v.Candidate == "" {
return fmt.Sprintf("%s: %s", v.Entry, v.Detail)
}
return fmt.Sprintf("%s -> candidate %q: %s", v.Entry, v.Candidate, v.Detail)
}
func formatViolations(violations []candidateViolation) string {
lines := make([]string, 0, len(violations))
for _, v := range violations {
lines = append(lines, " "+v.String())
}
return "\n" + strings.Join(lines, "\n")
}
func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel {
byName := make(map[string]gallery.GalleryModel, len(entries))
for _, e := range entries {
byName[e.Name] = e
}
return byName
}
// baseCandidate mirrors the implicit last-resort candidate the resolver
// synthesizes from the entry itself, so lint measures the same floor the
// installer will.
func baseCandidate(e gallery.GalleryModel) gallery.Candidate {
return gallery.Candidate{Model: e.Name, Capability: e.Capability, MinVRAM: e.MinVRAM}
}
// checkCandidateReferences verifies every candidate names an existing entry
// that declares no candidates of its own. Resolution is a single pass, so a
// nested reference would silently ignore the inner list.
func checkCandidateReferences(entries []gallery.GalleryModel) []candidateViolation {
byName := indexEntriesByName(entries)
var violations []candidateViolation
for _, e := range entries {
if !e.HasCandidates() {
continue
}
for _, c := range e.Candidates {
target, ok := byName[c.Model]
if !ok {
violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "references unknown model"})
continue
}
if target.HasCandidates() {
violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "references an entry that declares candidates of its own; nesting is not allowed"})
}
}
}
return violations
}
// checkCandidateFloors verifies every declared candidate carries a parseable
// VRAM floor. A candidate is an UPGRADE over the entry, so it must say what it
// costs; a floorless one would capture every host and make the entry's own
// payload unreachable.
func checkCandidateFloors(entries []gallery.GalleryModel) []candidateViolation {
var violations []candidateViolation
for _, e := range entries {
if !e.HasCandidates() {
continue
}
for _, c := range e.Candidates {
_, declared, err := c.EffectiveMinVRAM()
switch {
case err != nil:
violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "has a bad min_vram: " + err.Error()})
case !declared:
violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "needs a min_vram; the nightly job should have inferred one"})
}
}
}
return violations
}
// checkBaseFloor verifies the entry's own floor sits strictly below every
// candidate's. The entry is the last element of its own candidate list, so a
// base floor at or above a candidate's makes that candidate dead: any host
// clearing it already cleared the base, and first-match never gets that far.
func checkBaseFloor(entries []gallery.GalleryModel) []candidateViolation {
var violations []candidateViolation
for _, e := range entries {
if !e.HasCandidates() {
continue
}
base := baseCandidate(e)
baseFloor, _, err := base.EffectiveMinVRAM()
if err != nil {
violations = append(violations, candidateViolation{Entry: e.Name, Detail: "has a bad min_vram: " + err.Error()})
continue
}
for _, c := range e.Candidates {
floor, declared, cerr := c.EffectiveMinVRAM()
if cerr != nil || !declared {
// checkCandidateFloors owns reporting those.
continue
}
if baseFloor >= floor {
violations = append(violations, candidateViolation{
Entry: e.Name,
Candidate: c.Model,
Detail: fmt.Sprintf("sits at or below the entry's own %d byte floor, so it can never be reached",
baseFloor),
})
}
}
}
return violations
}
// checkCandidateCapabilities verifies entries and candidates only name
// capabilities the system can actually report, since the resolver compares
// them exactly.
func checkCandidateCapabilities(entries []gallery.GalleryModel) []candidateViolation {
var violations []candidateViolation
for _, e := range entries {
if !e.HasCandidates() {
continue
}
if e.Capability != "" && !knownCapabilities[e.Capability] {
violations = append(violations, candidateViolation{
Entry: e.Name,
Detail: fmt.Sprintf("uses unknown capability %q", e.Capability),
})
}
for _, c := range e.Candidates {
if c.Capability == "" {
continue
}
if !knownCapabilities[c.Capability] {
violations = append(violations, candidateViolation{
Entry: e.Name,
Candidate: c.Model,
Detail: fmt.Sprintf("uses unknown capability %q", c.Capability),
})
}
}
}
return violations
}
// checkCandidateOrdering verifies no candidate is shadowed by an earlier one.
// Selection is first-match over the authored order, so a candidate is dead
// whenever an earlier one matches every host this one would.
//
// Two shapes cause that. Within a single capability group the groups are
// mutually exclusive, so only a floor rising above an earlier floor is
// unreachable. A candidate with an EMPTY capability matches every host and so
// dominates ACROSS groups: every later candidate whose floor is at or above
// the running minimum unconditional floor is dead, whatever capability it asks
// for. Tracking that running minimum subsumes the same-group check for the
// empty capability.
func checkCandidateOrdering(entries []gallery.GalleryModel) []candidateViolation {
var violations []candidateViolation
for _, e := range entries {
if !e.HasCandidates() {
continue
}
previous := map[string]uint64{}
var unconditionalFloor uint64
haveUnconditional := false
for _, c := range e.Candidates {
floor, declared, err := c.EffectiveMinVRAM()
if err != nil || !declared {
// Parse errors and absent floors belong to checkCandidateFloors.
continue
}
switch {
case haveUnconditional && floor >= unconditionalFloor:
violations = append(violations, candidateViolation{
Entry: e.Name,
Candidate: c.Model,
Detail: fmt.Sprintf("is shadowed by an earlier candidate that matches any host at a %d byte floor, so it can never be reached",
unconditionalFloor),
})
case c.Capability != "":
if prior, seen := previous[c.Capability]; seen && floor > prior {
violations = append(violations, candidateViolation{
Entry: e.Name,
Candidate: c.Model,
Detail: "raises the VRAM floor after a lower one in the same capability group, so it can never be reached",
})
}
}
if c.Capability == "" && (!haveUnconditional || floor < unconditionalFloor) {
unconditionalFloor = floor
haveUnconditional = true
}
previous[c.Capability] = floor
}
}
return violations
}
// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The
// index carries well over a thousand entries, so re-parsing it per spec is
// pure overhead.
var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) {
data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml"))
if err != nil {
return nil, err
}
var entries []gallery.GalleryModel
if err := yaml.Unmarshal(data, &entries); err != nil {
return nil, err
}
return entries, nil
})
func plainEntry(name, url string) gallery.GalleryModel {
e := gallery.GalleryModel{}
e.Name = name
e.URL = url
return e
}
func entryWithCandidates(name, url, minVRAM string, candidates ...gallery.Candidate) gallery.GalleryModel {
e := gallery.GalleryModel{Candidates: candidates, MinVRAM: minVRAM}
e.Name = name
e.URL = url
return e
}
// candidateFixture builds an entry with candidates plus the entries it
// references. Synthetic fixtures keep the invariant logic covered however few
// entries in gallery/index.yaml declare candidates; without them every
// index-driven spec below is a no-op that passes while checking nothing.
func candidateFixture(base gallery.GalleryModel, referenced ...gallery.GalleryModel) []gallery.GalleryModel {
return append([]gallery.GalleryModel{base}, referenced...)
}
var _ = Describe("gallery candidate lint helpers", func() {
// An entry declares candidates solely by carrying the key. GalleryBackend.IsMeta()
// has deliberately different semantics (it requires an EMPTY uri), so a
// well-meaning alignment of the two would make every helper skip every
// entry and pass silently. Assert the distinction directly.
It("treats an entry with candidates as such even though it has a url", func() {
Expect(entryWithCandidates("base", "u://base", "2GiB", gallery.Candidate{Model: "big"}).HasCandidates()).To(BeTrue())
Expect(plainEntry("big", "u://big").HasCandidates()).To(BeFalse())
})
It("passes every invariant on a valid entry", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "4GiB",
gallery.Candidate{Model: "big", Capability: "nvidia", MinVRAM: "24GiB"},
gallery.Candidate{Model: "mid", Capability: "nvidia", MinVRAM: "12GiB"},
gallery.Candidate{Model: "metal-big", Capability: "metal", MinVRAM: "32GiB"},
gallery.Candidate{Model: "small", MinVRAM: "8GiB"},
),
plainEntry("big", "u://big"),
plainEntry("mid", "u://mid"),
plainEntry("metal-big", "u://metal-big"),
plainEntry("small", "u://small"),
)
Expect(checkCandidateReferences(entries)).To(BeEmpty())
Expect(checkCandidateFloors(entries)).To(BeEmpty())
Expect(checkBaseFloor(entries)).To(BeEmpty())
Expect(checkCandidateCapabilities(entries)).To(BeEmpty())
Expect(checkCandidateOrdering(entries)).To(BeEmpty())
})
It("passes every invariant on an entry that declares no floor of its own", func() {
// A floorless entry is the weakest possible base, below every
// candidate, which is exactly the position the base should hold.
entries := candidateFixture(
entryWithCandidates("base", "u://base", "",
gallery.Candidate{Model: "big", MinVRAM: "24GiB"},
),
plainEntry("big", "u://big"),
)
Expect(checkCandidateFloors(entries)).To(BeEmpty())
Expect(checkBaseFloor(entries)).To(BeEmpty())
Expect(checkCandidateOrdering(entries)).To(BeEmpty())
})
Describe("checkBaseFloor", func() {
It("flags a candidate whose floor sits below the entry's own", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "20GiB",
gallery.Candidate{Model: "big", MinVRAM: "8GiB"},
),
plainEntry("big", "u://big"),
)
violations := checkBaseFloor(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("big"))
Expect(violations[0].Detail).To(ContainSubstring("at or below the entry's own"))
})
It("flags a candidate whose floor merely equals the entry's own", func() {
// Equal floors make the candidate unreachable just as surely: every
// host clearing it clears the base, and the base is checked first
// only in the sense that first-match never reaches this candidate.
entries := candidateFixture(
entryWithCandidates("base", "u://base", "8GiB",
gallery.Candidate{Model: "big", MinVRAM: "8GiB"},
),
plainEntry("big", "u://big"),
)
violations := checkBaseFloor(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("big"))
})
It("flags an entry whose own min_vram cannot be parsed", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "eight gigs",
gallery.Candidate{Model: "big", MinVRAM: "24GiB"},
),
plainEntry("big", "u://big"),
)
violations := checkBaseFloor(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Entry).To(Equal("base"))
Expect(violations[0].Candidate).To(BeEmpty())
Expect(violations[0].Detail).To(ContainSubstring("bad min_vram"))
})
})
Describe("checkCandidateOrdering", func() {
It("flags a candidate shadowed by an earlier unconditional candidate", func() {
// "a" matches any host at 8GiB, so the nvidia candidate at 20GiB is
// dead: every nvidia host clearing 20GiB already cleared 8GiB.
entries := candidateFixture(
entryWithCandidates("base", "u://base", "2GiB",
gallery.Candidate{Model: "a", MinVRAM: "8GiB"},
gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"},
),
plainEntry("a", "u://a"), plainEntry("b", "u://b"),
)
violations := checkCandidateOrdering(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("b"))
Expect(violations[0].Detail).To(ContainSubstring("shadowed by an earlier candidate that matches any host"))
})
It("flags a floor inversion inside one capability group", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "2GiB",
gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"},
gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"},
),
plainEntry("a", "u://a"), plainEntry("b", "u://b"),
)
violations := checkCandidateOrdering(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("b"))
Expect(violations[0].Detail).To(ContainSubstring("same capability group"))
})
It("keeps distinct capability groups independent", func() {
// A high metal floor after a low nvidia floor is fine: no host
// reports both capabilities.
entries := candidateFixture(
entryWithCandidates("base", "u://base", "2GiB",
gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"},
gallery.Candidate{Model: "b", Capability: "metal", MinVRAM: "32GiB"},
),
plainEntry("a", "u://a"), plainEntry("b", "u://b"),
)
Expect(checkCandidateOrdering(entries)).To(BeEmpty())
})
})
Describe("checkCandidateFloors", func() {
It("flags a candidate with no floor", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "2GiB",
gallery.Candidate{Model: "a", Capability: "nvidia"},
),
plainEntry("a", "u://a"),
)
violations := checkCandidateFloors(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("a"))
Expect(violations[0].Detail).To(ContainSubstring("needs a min_vram"))
})
It("flags a candidate whose floor cannot be parsed", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "2GiB",
gallery.Candidate{Model: "a", MinVRAM: "lots"},
),
plainEntry("a", "u://a"),
)
violations := checkCandidateFloors(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("a"))
Expect(violations[0].Detail).To(ContainSubstring("bad min_vram"))
})
})
Describe("checkCandidateReferences", func() {
It("flags a candidate naming an entry that does not exist", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "2GiB",
gallery.Candidate{Model: "ghost", MinVRAM: "20GiB"},
),
plainEntry("a", "u://a"),
)
violations := checkCandidateReferences(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("ghost"))
Expect(violations[0].Detail).To(ContainSubstring("unknown model"))
})
It("flags a candidate naming an entry that declares candidates itself", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "2GiB",
gallery.Candidate{Model: "nested", MinVRAM: "20GiB"},
),
entryWithCandidates("nested", "u://nested", "4GiB", gallery.Candidate{Model: "a", MinVRAM: "30GiB"}),
plainEntry("a", "u://a"),
)
violations := checkCandidateReferences(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("nested"))
Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed"))
})
})
Describe("checkCandidateCapabilities", func() {
It("flags a capability the system can never report", func() {
entries := candidateFixture(
entryWithCandidates("base", "u://base", "2GiB",
gallery.Candidate{Model: "a", Capability: "cuda", MinVRAM: "20GiB"},
),
plainEntry("a", "u://a"),
)
violations := checkCandidateCapabilities(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Candidate).To(Equal("a"))
Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "cuda"`))
})
It("flags an unknown capability on the entry itself", func() {
entry := entryWithCandidates("base", "u://base", "2GiB", gallery.Candidate{Model: "a", MinVRAM: "20GiB"})
entry.Capability = "NVIDIA"
entries := candidateFixture(entry, plainEntry("a", "u://a"))
violations := checkCandidateCapabilities(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Entry).To(Equal("base"))
Expect(violations[0].Candidate).To(BeEmpty())
Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "NVIDIA"`))
})
})
})
var _ = Describe("gallery/index.yaml candidate invariants", Ordered, func() {
var entries []gallery.GalleryModel
BeforeAll(func() {
var err error
entries, err = loadGalleryIndex()
Expect(err).ToNot(HaveOccurred())
// A truncated or emptied index unmarshals cleanly and would make every
// spec below vacuously pass.
Expect(entries).ToNot(BeEmpty())
})
It("references only existing entries that declare no candidates themselves", func() {
v := checkCandidateReferences(entries)
Expect(v).To(BeEmpty(), formatViolations(v))
})
It("gives every candidate a VRAM floor", func() {
v := checkCandidateFloors(entries)
Expect(v).To(BeEmpty(), formatViolations(v))
})
It("keeps every entry's own floor below its candidates'", func() {
v := checkBaseFloor(entries)
Expect(v).To(BeEmpty(), formatViolations(v))
})
It("uses only capabilities the system can report", func() {
v := checkCandidateCapabilities(entries)
Expect(v).To(BeEmpty(), formatViolations(v))
})
It("orders candidates so no candidate is shadowed by an earlier one", func() {
v := checkCandidateOrdering(entries)
Expect(v).To(BeEmpty(), formatViolations(v))
})
})

8
core/gallery/model_artifacts.go
View File

@@ -32,10 +32,10 @@ func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption {
}
}
// WithVariant pins a gallery entry to a specific candidate by name, bypassing
// hardware-based resolution. The entry's own name is a valid pin, since the
// entry is itself the last-resort candidate. Ignored for entries that declare
// no candidates.
// WithVariant pins a gallery entry to a specific variant by name, bypassing
// hardware-based selection. The entry's own name is a valid pin, since the
// entry is itself the last resort. Ignored for entries that declare no
// variants.
func WithVariant(variant string) InstallOption {
return func(options *installOptions) {
options.variant = variant

172
core/gallery/models.go
View File

@@ -17,6 +17,7 @@ import (
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/mudler/xlog"
"gopkg.in/yaml.v3"
@@ -61,7 +62,7 @@ type ModelConfig struct {
Files []File `yaml:"files"`
PromptTemplates []PromptTemplate `yaml:"prompt_templates"`
// The fields below record how an entry carrying candidates was resolved,
// The fields below record how an entry carrying variants was resolved,
// so a reinstall or upgrade can honor the same pin and so operators can
// see which variant a stable model name is actually backed by.
EntryName string `yaml:"entry_name,omitempty"`
@@ -80,24 +81,38 @@ type PromptTemplate struct {
Content string `yaml:"content"`
}
// effectiveCandidates returns the candidate list resolution actually runs over:
// the declared upgrades followed by the entry itself.
// variantOptions turns an entry's declared variants into selectable options,
// resolving each referenced gallery entry so the selector gets the backend it
// filters on, and appends the entry itself as the base option.
//
// The entry is appended rather than special-cased so there is exactly one
// selection path. Being last makes it the last resort, which is what keeps the
// entry always installable: the list can never be exhausted without reaching it.
func effectiveCandidates(entry *GalleryModel) []Candidate {
candidates := make([]Candidate, 0, len(entry.Candidates)+1)
candidates = append(candidates, entry.Candidates...)
return append(candidates, Candidate{
Model: entry.Name,
Capability: entry.Capability,
MinVRAM: entry.MinVRAM,
})
// The base is appended rather than special-cased downstream so there is exactly
// one selection path, and so an operator can pin the entry's own name to
// decline an upgrade their hardware would otherwise take.
func variantOptions(models []*GalleryModel, entry *GalleryModel) ([]VariantOption, error) {
options := make([]VariantOption, 0, len(entry.Variants)+1)
for _, v := range entry.Variants {
// Every referenced entry is looked up here rather than lazily after
// selection, because the backend that decides whether a variant is even
// eligible lives on that entry, not on the variant.
target := FindGalleryElement(models, v.Model)
if target == nil {
return nil, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", entry.Name, v.Model)
}
if target.HasVariants() {
return nil, fmt.Errorf("model %q references variant %q which declares variants of its own; resolution is a single pass, so those would be silently ignored", entry.Name, v.Model)
}
options = append(options, VariantOption{Variant: v, Backend: target.Backend})
}
return append(options, VariantOption{
Variant: Variant{Model: entry.Name, MinMemory: entry.MinMemory},
Backend: entry.Backend,
IsBase: true,
}), nil
}
// ResolveVariant picks the gallery entry to install for a host, from the
// upgrades an entry declares plus the entry itself, and returns it renamed to
// variants an entry declares plus the entry itself, and returns it renamed to
// the entry's name and carrying the entry's presentation metadata.
//
// Why the metadata split: the payload (url, config_file, files, overrides)
@@ -106,53 +121,44 @@ func effectiveCandidates(entry *GalleryModel) []Candidate {
// from the entry the user asked for, so the installed model presents as that
// model rather than as one of its variants.
//
// The base entry always resolves. When even its own predicates fall short this
// warns and installs it regardless, because the entry is a complete entry that
// every older LocalAI release installs unconditionally; refusing it here would
// make the gallery behave worse the newer the client is.
func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Candidate, error) {
candidates := effectiveCandidates(entry)
base := candidates[len(candidates)-1]
candidate, err := ResolveCandidate(candidates, env, pin)
// The base entry always resolves. When even its own requirement falls short
// this installs it regardless, because the entry is a complete entry that every
// older LocalAI release installs unconditionally; refusing it here would make
// the gallery behave worse the newer the client is.
func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Variant, error) {
options, err := variantOptions(models, entry)
if err != nil {
if !errors.Is(err, ErrNoCandidateMatch) {
return nil, Candidate{}, fmt.Errorf("resolving variant for model %q: %w", entry.Name, err)
}
xlog.Warn("This system does not meet the requirements this model declares for itself; installing it anyway because it is the last resort",
"model", entry.Name, "capability", env.Capability, "available_vram", env.VRAM, "reason", err)
candidate = base
return nil, Variant{}, err
}
selection, err := SelectVariant(options, env, pin)
if err != nil {
return nil, Variant{}, fmt.Errorf("resolving variant for model %q: %w", entry.Name, err)
}
selected := selection.Option
if selection.FellBackToBase && len(entry.Variants) > 0 {
xlog.Warn("No declared variant of this model fits this system; installing the entry's own build",
"model", entry.Name, "available_memory", env.AvailableMemory, "reasons", strings.Join(selection.Reasons, "; "))
}
// A pin is an operator override and deliberately bypasses the hardware
// checks, but a silent bypass makes a later out-of-memory failure
// impossible to trace back to the pin, so it is recorded loudly here.
// It is warned about only once the pin is known to name a real, installable
// entry, otherwise a pin naming a listed-but-nonexistent variant would warn
// about VRAM and then fail for an entirely unrelated reason.
warnPin := func() {
if pin == "" {
return
}
if floor, declared, verr := candidate.EffectiveMinVRAM(); verr == nil && declared && env.VRAM < floor {
xlog.Warn("Pinned model variant declares more VRAM than this system reports; installing anyway because the pin overrides hardware resolution",
"model", entry.Name, "variant", candidate.Model, "required_vram", floor, "available_vram", env.VRAM)
if pin != "" {
if need, known, verr := selected.Variant.EffectiveMinMemory(); verr == nil && known && env.AvailableMemory < need {
xlog.Warn("Pinned model variant declares more memory than this system reports; installing anyway because the pin overrides hardware resolution",
"model", entry.Name, "variant", selected.Variant.Model, "required_memory", need, "available_memory", env.AvailableMemory)
}
}
// Resolving to the base means installing the entry's own payload, which is
// the entry itself; there is no second entry to look up.
// Selecting the base means installing the entry's own payload, which is the
// entry itself; there is no second entry to look up.
source := entry
if candidate.Model != entry.Name {
source = FindGalleryElement(models, candidate.Model)
if source == nil {
return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", entry.Name, candidate.Model)
}
if source.HasCandidates() {
return nil, Candidate{}, fmt.Errorf("model %q references variant %q which declares candidates of its own; resolution is a single pass, so those would be silently ignored", entry.Name, candidate.Model)
}
if !selected.IsBase {
// variantOptions already proved this lookup succeeds.
source = FindGalleryElement(models, selected.Variant.Model)
}
warnPin()
resolved := *source
resolved.Name = entry.Name
@@ -162,9 +168,8 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv,
// The resolved entry is a concrete install target, so it must not carry the
// selection fields any more; leaving them would let a second pass resolve
// the already-resolved entry all over again.
resolved.Candidates = nil
resolved.Capability = ""
resolved.MinVRAM = ""
resolved.Variants = nil
resolved.MinMemory = ""
// The struct copy above is shallow, so every reference-typed field still
// aliases the gallery's own entries. The install path mutates Overrides in
@@ -185,7 +190,33 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv,
resolved.URLs = slices.Clone(entry.URLs)
resolved.Tags = slices.Clone(entry.Tags)
return &resolved, candidate, nil
return &resolved, selected.Variant, nil
}
// noGPUDetected is what SystemState.DetectedCapability() reports when no
// usable GPU was found. The constant is unexported in pkg/system.
const noGPUDetected = "default"
// availableModelMemory reports how much memory a model may occupy on this host.
//
// With a usable GPU the model lives in VRAM. Without one it lives in system
// RAM, read through xsysinfo because that path is cgroup-aware: under
// Kubernetes the container's limit, not the node's physical RAM, is what the
// model actually gets.
//
// An unreadable RAM figure yields 0, which drops every variant with a known
// requirement and installs the base. That is the safe direction: an unknown
// host should not be talked into a larger download.
func availableModelMemory(systemState *system.SystemState) uint64 {
if systemState.DetectedCapability() != noGPUDetected {
return systemState.VRAM
}
ram, err := xsysinfo.GetSystemRAMInfo()
if err != nil || ram == nil {
xlog.Warn("Could not read system RAM; treating this host as having no memory budget for variant selection", "error", err)
return 0
}
return ram.Total
}
// deepCopyStringMap copies a decoded YAML map so no part of the result, at any
@@ -276,7 +307,7 @@ func InstallModelFromGallery(
config.ResolvedVariant = record.ResolvedVariant
config.PinnedVariant = record.PinnedVariant
// The variant's own name would otherwise be persisted here, which
// contradicts the whole point of candidates: the model is known by
// contradicts the whole point of variants: the model is known by
// the entry's stable name regardless of which variant backs it.
config.Name = record.EntryName
}
@@ -325,10 +356,10 @@ func InstallModelFromGallery(
return fmt.Errorf("no model found with name %q", name)
}
// An entry without candidates is installed directly. An entry with them is
// still installable as-is; resolution below only decides whether one of its
// declared upgrades fits this host better.
if !model.HasCandidates() {
// An entry without variants is installed directly. An entry with them is
// still installable as-is; selection below only decides whether one of its
// declared alternatives suits this host better.
if !model.HasVariants() {
return applyModel(model, nil)
}
@@ -354,22 +385,31 @@ func InstallModelFromGallery(
}
env := ResolveEnv{
Capability: systemState.DetectedCapability(),
VRAM: systemState.VRAM,
AvailableMemory: availableModelMemory(systemState),
// The whole hardware gate. IsBackendCompatible already derives
// Darwin-only, NVIDIA-only, ROCm-only and SYCL-only from the backend
// name, so a gallery author never has to describe hardware.
//
// The uri argument is deliberately empty: it exists for backend OCI
// images, and passing a model's gallery url here would let an unrelated
// substring in a download link decide hardware compatibility.
BackendCompatible: func(backend string) bool {
return systemState.IsBackendCompatible(backend, "")
},
}
resolved, candidate, err := ResolveVariant(models, model, env, pin)
resolved, variant, err := ResolveVariant(models, model, env, pin)
if err != nil {
return err
}
xlog.Info("Resolved model to variant",
"model", model.Name, "variant", candidate.Model,
"capability", env.Capability, "vram", env.VRAM, "pinned", pin != "")
"model", model.Name, "variant", variant.Model,
"available_memory", env.AvailableMemory, "pinned", pin != "")
return applyModel(resolved, &ModelConfig{
EntryName: model.Name,
ResolvedVariant: candidate.Model,
ResolvedVariant: variant.Model,
PinnedVariant: pin,
})
}

38
core/gallery/models_types.go
View File

@@ -15,19 +15,20 @@ type GalleryModel struct {
ConfigFile map[string]any `json:"config_file,omitempty" yaml:"config_file,omitempty"`
// Overrides are used to override the configuration of the model located at URL
Overrides map[string]any `json:"overrides,omitempty" yaml:"overrides,omitempty"`
// Candidates is an optional ordered list of hardware-gated UPGRADES over
// this entry. The entry itself is always the last-resort candidate, so an
// entry carrying candidates stays a complete, installable entry and older
// LocalAI releases, which drop this key, install it exactly as before.
Candidates []Candidate `json:"candidates,omitempty" yaml:"candidates,omitempty"`
// Capability, when set, is the host capability this entry's own payload
// prefers. It is advisory for the base entry: an unmet capability warns
// rather than refusing, because the base always installs.
Capability string `json:"capability,omitempty" yaml:"capability,omitempty"`
// MinVRAM is this entry's own VRAM floor, in the same form as a
// candidate's (e.g. "2GiB"). It positions the entry within its own
// candidate list and is likewise advisory: falling short only warns.
MinVRAM string `json:"min_vram,omitempty" yaml:"min_vram,omitempty"`
// Variants is an optional, UNORDERED list of alternative builds of the same
// model (other backends such as MLX or vLLM, other quantizations) that the
// installer may pick instead of this entry's own payload. Authoring is
// deliberately dumb: name the models, and the selector works out which one
// this host should get.
//
// The entry itself is always the last resort, so an entry carrying variants
// stays a complete, installable entry and older LocalAI releases, which drop
// this key, install it exactly as before.
Variants []Variant `json:"variants,omitempty" yaml:"variants,omitempty"`
// MinMemory is this entry's own memory requirement (e.g. "2GiB"), in the
// same form as a variant's. It is advisory for the base entry: falling short
// only warns, because the base always installs.
MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"`
}
func (m *GalleryModel) GetInstalled() bool {
@@ -58,12 +59,11 @@ func (m GalleryModel) ID() string {
return fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name)
}
// HasCandidates reports whether this entry declares hardware-gated upgrades
// over its own payload. It says nothing about the entry being installable:
// an entry with candidates is a complete entry that can always be installed
// as-is.
func (m GalleryModel) HasCandidates() bool {
return len(m.Candidates) > 0
// HasVariants reports whether this entry declares alternative builds of itself.
// It says nothing about the entry being installable: an entry with variants is
// a complete entry that can always be installed as-is.
func (m GalleryModel) HasVariants() bool {
return len(m.Variants) > 0
}
func (m *GalleryModel) GetTags() []string {

84
core/gallery/resolve_candidate.go
View File

@@ -1,84 +0,0 @@
package gallery
import (
"errors"
"fmt"
"strings"
)
var (
// ErrNoCandidateMatch is returned when no candidate satisfies the host.
ErrNoCandidateMatch = errors.New("no model variant matches this system")
// ErrPinNotFound is returned when a pinned variant is absent from the list.
ErrPinNotFound = errors.New("pinned model variant not found")
)
// ResolveEnv describes the host properties a candidate is matched against.
//
// It is a plain struct rather than a *SystemState so the resolver stays pure
// and testable across hardware shapes without touching the machine.
type ResolveEnv struct {
// Capability is the raw value from SystemState.DetectedCapability().
Capability string
// VRAM is total available VRAM in bytes, already capped by the operator
// VRAM budget because the cap is applied inside detection.
VRAM uint64
}
// ResolveCandidate returns the first candidate the host satisfies, or the
// pinned candidate when pin is non-empty.
//
// Why first-match rather than best-match: the list order is the policy. It is
// authored deliberately and lint-checked, which keeps selection explainable
// (you can read the list top to bottom) and keeps fallback expressible without
// a scoring function nobody can predict.
func ResolveCandidate(candidates []Candidate, env ResolveEnv, pin string) (Candidate, error) {
if pin != "" {
for _, c := range candidates {
if strings.EqualFold(c.Model, pin) {
return c, nil
}
}
return Candidate{}, fmt.Errorf("%w: %q is not among the %d variants of this model", ErrPinNotFound, pin, len(candidates))
}
reasons := make([]string, 0, len(candidates))
for _, c := range candidates {
if c.Capability != "" && c.Capability != env.Capability {
reasons = append(reasons, fmt.Sprintf("%s requires capability %q", c.Model, c.Capability))
continue
}
floor, declared, err := c.EffectiveMinVRAM()
if err != nil {
return Candidate{}, err
}
// A candidate with no declared floor states no requirement and
// passes. Lint guarantees only the final last-resort candidate is
// in that position for the official gallery.
if declared && env.VRAM < floor {
reasons = append(reasons, fmt.Sprintf("%s needs %s VRAM", c.Model, humanBytes(floor)))
continue
}
return c, nil
}
return Candidate{}, fmt.Errorf(
"%w: capability %q with %s VRAM available; candidates: %s",
ErrNoCandidateMatch, env.Capability, humanBytes(env.VRAM), strings.Join(reasons, "; "),
)
}
func humanBytes(b uint64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%dB", b)
}
div, exp := uint64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f%ciB", float64(b)/float64(div), "KMGTPE"[exp])
}

108
core/gallery/resolve_candidate_test.go
View File

@@ -1,108 +0,0 @@
package gallery_test
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/gallery"
)
var _ = Describe("Candidate.EffectiveMinVRAM", func() {
It("reports no floor when neither authored nor inferred", func() {
c := gallery.Candidate{Model: "x"}
bytes, declared, err := c.EffectiveMinVRAM()
Expect(err).ToNot(HaveOccurred())
Expect(declared).To(BeFalse())
Expect(bytes).To(Equal(uint64(0)))
})
It("uses the inferred value when nothing is authored", func() {
c := gallery.Candidate{Model: "x", InferredMinVRAM: "6GiB"}
bytes, declared, err := c.EffectiveMinVRAM()
Expect(err).ToNot(HaveOccurred())
Expect(declared).To(BeTrue())
Expect(bytes).To(Equal(uint64(6 * 1024 * 1024 * 1024)))
})
It("lets an authored value win over an inferred one", func() {
c := gallery.Candidate{Model: "x", MinVRAM: "20GiB", InferredMinVRAM: "6GiB"}
bytes, declared, err := c.EffectiveMinVRAM()
Expect(err).ToNot(HaveOccurred())
Expect(declared).To(BeTrue())
Expect(bytes).To(Equal(uint64(20 * 1024 * 1024 * 1024)))
})
It("errors on an unparseable floor rather than silently treating it as absent", func() {
c := gallery.Candidate{Model: "x", MinVRAM: "twenty gigs"}
_, _, err := c.EffectiveMinVRAM()
Expect(err).To(HaveOccurred())
})
})
var _ = Describe("ResolveCandidate", func() {
gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 }
list := []gallery.Candidate{
{Model: "m-mlx", Capability: "metal", MinVRAM: "16GiB"},
{Model: "m-vllm", Capability: "nvidia", MinVRAM: "20GiB"},
{Model: "m-q8", MinVRAM: "12GiB"},
{Model: "m-q4", MinVRAM: "6GiB"},
{Model: "m-q4-cpu"},
}
It("picks the capability-matched candidate when VRAM allows", func() {
got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(got.Model).To(Equal("m-vllm"))
})
It("skips a capability match whose VRAM floor does not fit", func() {
got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(8)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(got.Model).To(Equal("m-q4"))
})
It("ignores candidates whose capability does not match the host", func() {
got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "metal", VRAM: gib(24)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(got.Model).To(Equal("m-mlx"))
})
It("falls through to the unconstrained last resort on a tiny host", func() {
got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "default", VRAM: gib(2)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(got.Model).To(Equal("m-q4-cpu"))
})
It("honors a pin even when the host would have chosen otherwise", func() {
got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "m-q8")
Expect(err).ToNot(HaveOccurred())
Expect(got.Model).To(Equal("m-q8"))
})
It("fails loudly when the pin names an absent candidate", func() {
_, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "m-gone")
Expect(err).To(MatchError(gallery.ErrPinNotFound))
Expect(err.Error()).To(ContainSubstring("m-gone"))
})
It("reports the host state when nothing matches", func() {
constrained := []gallery.Candidate{{Model: "big", MinVRAM: "80GiB"}}
_, err := gallery.ResolveCandidate(constrained, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "")
Expect(err).To(MatchError(gallery.ErrNoCandidateMatch))
Expect(err.Error()).To(ContainSubstring("default"))
Expect(err.Error()).To(ContainSubstring("big"))
})
It("propagates an unparseable floor instead of treating it as unconstrained", func() {
bad := []gallery.Candidate{{Model: "bad", MinVRAM: "lots"}}
_, err := gallery.ResolveCandidate(bad, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "")
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("bad"))
})
It("rejects an empty candidate list", func() {
_, err := gallery.ResolveCandidate(nil, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "")
Expect(err).To(MatchError(gallery.ErrNoCandidateMatch))
})
})

162
core/gallery/resolve_variant.go Normal file
View File

@@ -0,0 +1,162 @@
package gallery
import (
"errors"
"fmt"
"sort"
"strings"
)
var (
// ErrNoVariantMatch is returned when nothing at all is selectable, which
// only happens for a caller that supplies no base option.
ErrNoVariantMatch = errors.New("no model variant matches this system")
// ErrPinNotFound is returned when a pinned variant is absent from the list.
ErrPinNotFound = errors.New("pinned model variant not found")
)
// VariantOption is a variant paired with the facts a host is matched against.
//
// The install layer builds these by looking the referenced entries up in the
// live gallery. The selector never touches the catalog, which is what keeps it
// pure and testable across hardware shapes without touching the machine.
type VariantOption struct {
Variant Variant
// Backend is the engine the referenced entry resolves to (e.g. "llama-cpp",
// "mlx", "vllm"). Hardware support is derived from this name alone, which is
// precisely why a gallery author never has to describe hardware.
Backend string
// IsBase marks the declaring entry's own payload. It is exempt from every
// filter and is the answer when nothing else survives, because the entry
// must stay installable on every host and for every client.
IsBase bool
}
// ResolveEnv describes the host a variant is selected for.
type ResolveEnv struct {
// AvailableMemory is what a model may occupy on this host: VRAM when a
// usable GPU was detected, system RAM otherwise. A model's footprint is
// roughly the same in either, so one number and one comparison cover both.
AvailableMemory uint64
// BackendCompatible reports whether a backend can run here. The install
// layer wires SystemState.IsBackendCompatible, which already derives
// Darwin-only, NVIDIA-only, ROCm-only and SYCL-only from the backend name.
//
// A nil func treats every backend as runnable, the right default for a
// caller with no view of the hardware.
BackendCompatible func(backend string) bool
}
func (e ResolveEnv) backendRuns(backend string) bool {
if e.BackendCompatible == nil {
return true
}
return e.BackendCompatible(backend)
}
// VariantSelection is the outcome of a selection pass.
type VariantSelection struct {
Option VariantOption
// FellBackToBase reports that no declared variant survived and the entry's
// own payload was chosen instead. Callers log this, because a host quietly
// taking the base when upgrades were on offer is worth being able to see.
FellBackToBase bool
// Reasons explains, one line per rejected variant, why it was dropped.
Reasons []string
}
// SelectVariant picks the option to install for a host.
//
// The algorithm, in order:
//
// 1. An explicit pin wins outright, fit or no fit. It is a deliberate operator
// override, so refusing it would defeat the point; the caller warns.
// 2. Variants whose backend cannot run here are dropped. This is the whole of
// the hardware gate, derived from the backend name.
// 3. Variants whose known memory requirement exceeds what the host has are
// dropped. A variant with an UNKNOWN requirement survives, because nothing
// proves it does not fit and refusing on a missing estimate would punish
// the entries the denormalizer has not reached yet.
// 4. The largest survivor wins. A bigger footprint is a higher quality
// quantization of the same model, so among things that fit, more is better.
// Unknown requirements rank last, so a proven fit always beats a guess.
// 5. With no survivor the base option wins. The base always installs; this
// never refuses.
func SelectVariant(options []VariantOption, env ResolveEnv, pin string) (VariantSelection, error) {
if pin != "" {
for _, o := range options {
if strings.EqualFold(o.Variant.Model, pin) {
return VariantSelection{Option: o}, nil
}
}
return VariantSelection{}, fmt.Errorf("%w: %q is not among the %d variants of this model", ErrPinNotFound, pin, len(options))
}
type ranked struct {
option VariantOption
memory uint64
known bool
}
var base *VariantOption
survivors := make([]ranked, 0, len(options))
reasons := make([]string, 0, len(options))
for i := range options {
o := options[i]
if o.IsBase {
base = &options[i]
continue
}
if !env.backendRuns(o.Backend) {
reasons = append(reasons, fmt.Sprintf("%s needs backend %q, which cannot run on this system", o.Variant.Model, o.Backend))
continue
}
memory, known, err := o.Variant.EffectiveMinMemory()
if err != nil {
return VariantSelection{}, err
}
if known && memory > env.AvailableMemory {
reasons = append(reasons, fmt.Sprintf("%s needs %s of memory", o.Variant.Model, humanBytes(memory)))
continue
}
survivors = append(survivors, ranked{option: o, memory: memory, known: known})
}
if len(survivors) > 0 {
// Stable so that variants with identical requirements keep their
// authored order, which is the only thing order still decides.
sort.SliceStable(survivors, func(i, j int) bool {
if survivors[i].known != survivors[j].known {
return survivors[i].known
}
return survivors[i].memory > survivors[j].memory
})
return VariantSelection{Option: survivors[0].option, Reasons: reasons}, nil
}
if base != nil {
return VariantSelection{Option: *base, FellBackToBase: true, Reasons: reasons}, nil
}
return VariantSelection{}, fmt.Errorf(
"%w: %s of memory available; variants: %s",
ErrNoVariantMatch, humanBytes(env.AvailableMemory), strings.Join(reasons, "; "),
)
}
func humanBytes(b uint64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%dB", b)
}
div, exp := uint64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f%ciB", float64(b)/float64(div), "KMGTPE"[exp])
}

273
core/gallery/resolve_variant_test.go Normal file
View File

@@ -0,0 +1,273 @@
package gallery_test
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/gallery"
)
var _ = Describe("Variant.EffectiveMinMemory", func() {
It("reports no requirement when neither authored nor inferred", func() {
v := gallery.Variant{Model: "x"}
size, known, err := v.EffectiveMinMemory()
Expect(err).ToNot(HaveOccurred())
Expect(known).To(BeFalse())
Expect(size).To(Equal(uint64(0)))
})
It("uses the inferred value when nothing is authored", func() {
v := gallery.Variant{Model: "x", InferredMinMemory: "6GiB"}
size, known, err := v.EffectiveMinMemory()
Expect(err).ToNot(HaveOccurred())
Expect(known).To(BeTrue())
Expect(size).To(Equal(uint64(6 * 1024 * 1024 * 1024)))
})
It("lets an authored value win over an inferred one", func() {
v := gallery.Variant{Model: "x", MinMemory: "20GiB", InferredMinMemory: "6GiB"}
size, known, err := v.EffectiveMinMemory()
Expect(err).ToNot(HaveOccurred())
Expect(known).To(BeTrue())
Expect(size).To(Equal(uint64(20 * 1024 * 1024 * 1024)))
})
It("errors on an unparseable figure rather than silently treating it as absent", func() {
v := gallery.Variant{Model: "x", MinMemory: "twenty gigs"}
_, _, err := v.EffectiveMinMemory()
Expect(err).To(HaveOccurred())
})
})
var _ = Describe("SelectVariant", func() {
gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 }
option := func(model, backend, minMemory string) gallery.VariantOption {
return gallery.VariantOption{
Variant: gallery.Variant{Model: model, MinMemory: minMemory},
Backend: backend,
}
}
base := func(model, minMemory string) gallery.VariantOption {
o := option(model, "llama-cpp", minMemory)
o.IsBase = true
return o
}
// linuxNvidia mirrors what SystemState.IsBackendCompatible does on a Linux
// box with an NVIDIA card: Darwin-only engines are out, everything else runs.
linuxNvidia := func(backend string) bool {
return backend != "mlx" && backend != "mlx-vlm"
}
Describe("hardware filtering", func() {
It("never selects a variant whose backend cannot run on this host", func() {
// The MLX build is both the largest and the only thing that would
// otherwise win, so nothing but the backend gate can reject it.
options := []gallery.VariantOption{
option("m-mlx-8bit", "mlx", "24GiB"),
option("m-gguf-q8", "llama-cpp", "12GiB"),
base("m-gguf-q4", "6GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{
AvailableMemory: gib(80),
BackendCompatible: linuxNvidia,
}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8"))
})
It("selects the same variant on a host whose backend gate does admit it", func() {
// The mirror image of the spec above, so the rejection is proven to
// come from the host and not from something intrinsic to the entry.
options := []gallery.VariantOption{
option("m-mlx-8bit", "mlx", "24GiB"),
option("m-gguf-q8", "llama-cpp", "12GiB"),
base("m-gguf-q4", "6GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{
AvailableMemory: gib(80),
BackendCompatible: func(string) bool { return true },
}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-mlx-8bit"))
})
It("treats every backend as runnable when the host cannot be inspected", func() {
options := []gallery.VariantOption{option("m-mlx-8bit", "mlx", "24GiB"), base("m-gguf-q4", "6GiB")}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(80)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-mlx-8bit"))
})
})
Describe("ranking", func() {
It("picks the largest variant that fits, not the first authored", func() {
// Authored smallest-first, so first-match would take m-q4 and any
// ranking that ignores size would too.
options := []gallery.VariantOption{
option("m-q4", "llama-cpp", "6GiB"),
option("m-q8", "llama-cpp", "12GiB"),
option("m-f16", "llama-cpp", "24GiB"),
base("m-base", "2GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-q8"))
Expect(selection.FellBackToBase).To(BeFalse())
})
It("picks the largest variant regardless of authored order", func() {
// Same set, authored largest-first. Order must make no difference at
// all, which is the entire point of dropping ordered first-match.
options := []gallery.VariantOption{
option("m-f16", "llama-cpp", "24GiB"),
option("m-q8", "llama-cpp", "12GiB"),
option("m-q4", "llama-cpp", "6GiB"),
base("m-base", "2GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-q8"))
})
It("prefers a known fit over a variant of unknown size", func() {
// An unknown requirement is a guess. It survives the filter, because
// nothing proves it does not fit, but it must never displace a
// variant that is known to fit.
options := []gallery.VariantOption{
option("m-unknown", "llama-cpp", ""),
option("m-q8", "llama-cpp", "12GiB"),
base("m-base", "2GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-q8"))
})
It("keeps a variant of unknown size rather than dropping it", func() {
options := []gallery.VariantOption{
option("m-unknown", "llama-cpp", ""),
base("m-base", "2GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-unknown"))
Expect(selection.FellBackToBase).To(BeFalse())
})
It("admits a variant needing exactly the memory available", func() {
options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", "2GiB")}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(12)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-q8"))
})
})
Describe("falling back to the base", func() {
It("selects the base when nothing else fits", func() {
options := []gallery.VariantOption{
option("m-q8", "llama-cpp", "12GiB"),
option("m-f16", "llama-cpp", "24GiB"),
base("m-base", "2GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-base"))
Expect(selection.Option.IsBase).To(BeTrue())
Expect(selection.FellBackToBase).To(BeTrue())
Expect(selection.Reasons).To(HaveLen(2))
})
It("selects the base even when the base's own requirement is unmet", func() {
// There is nothing below the base, so refusing here would make an
// entry every older client installs fine uninstallable on newer ones.
options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", "2GiB")}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: 0}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-base"))
})
It("explains why each variant was rejected", func() {
options := []gallery.VariantOption{
option("m-mlx", "mlx", "8GiB"),
option("m-f16", "llama-cpp", "24GiB"),
base("m-base", "2GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{
AvailableMemory: gib(4),
BackendCompatible: linuxNvidia,
}, "")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Reasons).To(ContainElement(ContainSubstring("cannot run on this system")))
Expect(selection.Reasons).To(ContainElement(ContainSubstring("24.0GiB")))
})
It("errors when the caller supplies no base at all", func() {
options := []gallery.VariantOption{option("m-f16", "llama-cpp", "24GiB")}
_, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "")
Expect(err).To(MatchError(gallery.ErrNoVariantMatch))
})
})
Describe("explicit selection", func() {
It("honors a pin the hardware would never have chosen", func() {
options := []gallery.VariantOption{
option("m-mlx", "mlx", "64GiB"),
base("m-base", "2GiB"),
}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{
AvailableMemory: gib(4),
BackendCompatible: linuxNvidia,
}, "m-mlx")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-mlx"))
})
It("honors a pin naming the base, declining an upgrade that fits", func() {
options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-base")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-base"))
})
It("matches a pin case-insensitively", func() {
options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")}
selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "M-F16")
Expect(err).ToNot(HaveOccurred())
Expect(selection.Option.Variant.Model).To(Equal("m-f16"))
})
It("fails loudly when the pin names nothing in the list", func() {
options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")}
_, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-gone")
Expect(err).To(MatchError(gallery.ErrPinNotFound))
Expect(err.Error()).To(ContainSubstring("m-gone"))
})
})
It("propagates an unparseable figure instead of treating it as unconstrained", func() {
options := []gallery.VariantOption{option("bad", "llama-cpp", "lots"), base("m-base", "2GiB")}
_, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(8)}, "")
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("bad"))
})
})

56
core/gallery/variant.go Normal file
View File

@@ -0,0 +1,56 @@
package gallery
import (
"fmt"
"github.com/mudler/LocalAI/pkg/vram"
)
// Variant is one option in a gallery entry's variant list. It references an
// existing gallery entry by name, and that is all an author has to write.
//
// Authored order carries no meaning. Selection filters out what this host
// cannot run and then ranks what is left, so hardware knowledge lives in the
// selector rather than being pushed onto whoever edits the gallery.
type Variant struct {
// Model is the name of a gallery entry that declares no variants of its own.
Model string `json:"model" yaml:"model"`
// MinMemory is the authored memory requirement (e.g. "20GiB"). It exists to
// override an inferred estimate that is known to be wrong; most variants
// should not need it.
//
// One number covers both VRAM and system RAM. A model's footprint is
// roughly the same wherever it lives, and the selector compares this against
// whichever of the two this host will actually use, so splitting it in two
// would only invite the pair to disagree.
MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"`
// The fields below are denormalized by the nightly job for display and
// lint. They are never authored by hand and never affect what gets
// installed, because installation reads the referenced entry live.
Backend string `json:"backend,omitempty" yaml:"backend,omitempty"`
Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"`
InferredMinMemory string `json:"inferred_min_memory,omitempty" yaml:"inferred_min_memory,omitempty"`
}
// EffectiveMinMemory returns this variant's memory requirement in bytes and
// whether one is known at all. An authored MinMemory wins over an inferred one,
// because a human who measured a real load knows more than a pre-download
// estimate does.
//
// An unknown requirement is not a zero requirement. Callers must not read the
// returned 0 as "fits anywhere"; it means nothing is known either way.
func (v Variant) EffectiveMinMemory() (uint64, bool, error) {
raw := v.MinMemory
if raw == "" {
raw = v.InferredMinMemory
}
if raw == "" {
return 0, false, nil
}
size, err := vram.ParseSizeString(raw)
if err != nil {
return 0, false, fmt.Errorf("variant %q has an unparseable min_memory %q: %w", v.Model, raw, err)
}
return size, true, nil
}

268
core/gallery/candidates_install_test.go → core/gallery/variants_install_test.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"os"
"path/filepath"
"runtime"
"dario.cat/mergo"
. "github.com/onsi/ginkgo/v2"
@@ -15,50 +16,51 @@ import (
"github.com/mudler/LocalAI/pkg/system"
)
var _ = Describe("GalleryModel candidate declarations", func() {
It("declares no candidates when the key is absent", func() {
var _ = Describe("GalleryModel variant declarations", func() {
It("declares no variants when the key is absent", func() {
m := gallery.GalleryModel{}
m.Name = "plain"
Expect(m.HasCandidates()).To(BeFalse())
Expect(m.HasVariants()).To(BeFalse())
})
It("declares candidates when the key is present", func() {
m := gallery.GalleryModel{Candidates: []gallery.Candidate{{Model: "x"}}}
Expect(m.HasCandidates()).To(BeTrue())
It("declares variants when the key is present", func() {
m := gallery.GalleryModel{Variants: []gallery.Variant{{Model: "x"}}}
Expect(m.HasVariants()).To(BeTrue())
})
It("parses an entry's own selection fields and its candidate list in order", func() {
It("parses an entry's own memory figure and its variant list", func() {
var m gallery.GalleryModel
err := yaml.Unmarshal([]byte(`
name: qwen3-8b-gguf-q4
url: "github:example/repo/qwen3-8b-gguf-q4.yaml@master"
min_vram: 6GiB
capability: default
candidates:
- model: qwen3-8b-vllm-awq
capability: nvidia
min_vram: 20GiB
- model: qwen3-8b-gguf-q8
min_vram: 10GiB
name: qwen3.6-27b
url: "github:example/repo/qwen3.6-27b.yaml@master"
min_memory: 4GiB
variants:
- model: qwen3.6-27b-mlx-8bit
- model: qwen3.6-27b-gguf-q8
min_memory: 28GiB
`), &m)
Expect(err).ToNot(HaveOccurred())
Expect(m.Name).To(Equal("qwen3-8b-gguf-q4"))
Expect(m.URL).To(Equal("github:example/repo/qwen3-8b-gguf-q4.yaml@master"))
Expect(m.MinVRAM).To(Equal("6GiB"))
Expect(m.Capability).To(Equal("default"))
Expect(m.HasCandidates()).To(BeTrue())
Expect(m.Candidates).To(HaveLen(2))
Expect(m.Candidates[0].Model).To(Equal("qwen3-8b-vllm-awq"))
Expect(m.Candidates[0].Capability).To(Equal("nvidia"))
Expect(m.Candidates[0].MinVRAM).To(Equal("20GiB"))
Expect(m.Candidates[1].Model).To(Equal("qwen3-8b-gguf-q8"))
Expect(m.Candidates[1].Capability).To(BeEmpty())
Expect(m.Name).To(Equal("qwen3.6-27b"))
Expect(m.URL).To(Equal("github:example/repo/qwen3.6-27b.yaml@master"))
Expect(m.MinMemory).To(Equal("4GiB"))
Expect(m.HasVariants()).To(BeTrue())
Expect(m.Variants).To(HaveLen(2))
// The first variant is nothing but a name, which is the shape authoring
// is meant to reach for.
Expect(m.Variants[0].Model).To(Equal("qwen3.6-27b-mlx-8bit"))
Expect(m.Variants[0].MinMemory).To(BeEmpty())
Expect(m.Variants[1].Model).To(Equal("qwen3.6-27b-gguf-q8"))
Expect(m.Variants[1].MinMemory).To(Equal("28GiB"))
})
})
var _ = Describe("ResolveVariant", func() {
gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 }
// runsEverything keeps these specs about resolution rather than about the
// hardware of whatever machine runs them.
runsEverything := func(string) bool { return true }
newModel := func(name, url, description, icon string) *gallery.GalleryModel {
m := &gallery.GalleryModel{}
m.Name = name
@@ -73,54 +75,67 @@ var _ = Describe("ResolveVariant", func() {
BeforeEach(func() {
upgrade := newModel("qwen3-8b-vllm-awq", "file://vllm.yaml", "AWQ variant", "vllm.png")
// The base is an ordinary, complete entry that happens to declare an
// upgrade over itself, which is the whole point of the design.
upgrade.Backend = "vllm"
// The base is an ordinary, complete entry that happens to list an
// alternative build of itself, which is the whole point of the design.
base = newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "Qwen3 8B Q4", "qwen.png")
base.Backend = "llama-cpp"
base.Tags = []string{"llm"}
base.MinVRAM = "6GiB"
base.Candidates = []gallery.Candidate{
{Model: "qwen3-8b-vllm-awq", Capability: "nvidia", MinVRAM: "20GiB"},
}
base.MinMemory = "6GiB"
base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}}
models = []*gallery.GalleryModel{upgrade, base}
})
It("installs a matching candidate's payload under the entry's name", func() {
resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "")
env := func(memory uint64) gallery.ResolveEnv {
return gallery.ResolveEnv{AvailableMemory: memory, BackendCompatible: runsEverything}
}
It("installs a fitting variant's payload under the entry's name", func() {
resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(24)), "")
Expect(err).ToNot(HaveOccurred())
Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq"))
Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq"))
Expect(resolved.Name).To(Equal("qwen3-8b-gguf-q4"))
Expect(resolved.URL).To(Equal("file://vllm.yaml"))
})
It("falls back to the entry's own payload when no candidate fits", func() {
resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "")
It("carries the referenced entry's backend into the compatibility check", func() {
// The variant itself says nothing about hardware, so the backend can
// only come from the entry it names. Rejecting exactly that backend must
// therefore be enough to rule the variant out.
resolved, variant, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{
AvailableMemory: gib(64),
BackendCompatible: func(backend string) bool { return backend != "vllm" },
}, "")
Expect(err).ToNot(HaveOccurred())
Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4"))
Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4"))
Expect(resolved.URL).To(Equal("file://gguf.yaml"))
})
It("installs the entry even when the host misses the entry's own floor", func() {
// There is nothing below the base, so refusing here would make an entry
// that every older client installs fine uninstallable on new ones.
resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(1)}, "")
It("falls back to the entry's own payload when no variant fits", func() {
resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(8)), "")
Expect(err).ToNot(HaveOccurred())
Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4"))
Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4"))
Expect(resolved.URL).To(Equal("file://gguf.yaml"))
})
It("installs the entry even when the host misses the entry's own requirement", func() {
resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(1)), "")
Expect(err).ToNot(HaveOccurred())
Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4"))
Expect(resolved.URL).To(Equal("file://gguf.yaml"))
})
It("strips the selection fields from the resolved entry", func() {
// A resolved entry is a concrete install target. Leaving the fields on
// it would let a second resolution pass fire on an already-resolved
// entry.
resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "")
// it would let a second selection pass fire on an already-resolved entry.
resolved, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "")
Expect(err).ToNot(HaveOccurred())
Expect(resolved.HasCandidates()).To(BeFalse())
Expect(resolved.MinVRAM).To(BeEmpty())
Expect(resolved.Capability).To(BeEmpty())
Expect(resolved.HasVariants()).To(BeFalse())
Expect(resolved.MinMemory).To(BeEmpty())
})
It("presents the entry's metadata, not the candidate's", func() {
resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "")
It("presents the entry's metadata, not the variant's", func() {
resolved, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "")
Expect(err).ToNot(HaveOccurred())
Expect(resolved.Description).To(Equal("Qwen3 8B Q4"))
Expect(resolved.Icon).To(Equal("qwen.png"))
@@ -128,20 +143,20 @@ var _ = Describe("ResolveVariant", func() {
})
It("honors a pin naming the entry itself", func() {
// The entry is the last element of its own candidate list, so its own
// name has to be a usable pin: it is how an operator declines an
// upgrade their hardware would otherwise take.
resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "qwen3-8b-gguf-q4")
// The entry is the last resort of its own variant list, so its own name
// has to be a usable pin: it is how an operator declines an upgrade
// their hardware would otherwise take.
resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(24)), "qwen3-8b-gguf-q4")
Expect(err).ToNot(HaveOccurred())
Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4"))
Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4"))
Expect(resolved.Name).To(Equal("qwen3-8b-gguf-q4"))
Expect(resolved.URL).To(Equal("file://gguf.yaml"))
})
It("honors a pin the hardware does not satisfy", func() {
resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(2)}, "qwen3-8b-vllm-awq")
resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(2)), "qwen3-8b-vllm-awq")
Expect(err).ToNot(HaveOccurred())
Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq"))
Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq"))
Expect(resolved.URL).To(Equal("file://vllm.yaml"))
})
@@ -149,7 +164,7 @@ var _ = Describe("ResolveVariant", func() {
models[0].Overrides = map[string]any{"f16": true}
models[0].AdditionalFiles = []gallery.File{{Filename: "a.bin"}}
resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "")
resolved, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "")
Expect(err).ToNot(HaveOccurred())
// The install path merges the caller's request overrides into this map in
@@ -172,7 +187,7 @@ var _ = Describe("ResolveVariant", func() {
// holds, which is the case most likely to alias it.
base.Overrides = map[string]any{"parameters": map[string]any{"model": "q4.gguf"}}
resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "")
resolved, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "")
Expect(err).ToNot(HaveOccurred())
resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf"
@@ -185,7 +200,7 @@ var _ = Describe("ResolveVariant", func() {
"stopwords": []any{"</s>"},
}
resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "")
resolved, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "")
Expect(err).ToNot(HaveOccurred())
// Cloning only the top level would leave this inner map shared with the
@@ -201,7 +216,7 @@ var _ = Describe("ResolveVariant", func() {
It("does not write the caller's overrides back into the gallery entry", func() {
models[0].Overrides = map[string]any{"parameters": map[string]any{"model": "real-variant.gguf"}}
resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "")
resolved, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "")
Expect(err).ToNot(HaveOccurred())
// This is exactly what the install path does with the caller's request
@@ -216,30 +231,30 @@ var _ = Describe("ResolveVariant", func() {
Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf"))
})
It("errors when a candidate references a missing entry", func() {
base.Candidates = []gallery.Candidate{{Model: "does-not-exist"}}
_, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "")
It("errors when a variant references a missing entry", func() {
base.Variants = []gallery.Variant{{Model: "does-not-exist"}}
_, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "")
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("does-not-exist"))
})
It("refuses a candidate that declares candidates of its own", func() {
It("refuses a variant that declares variants of its own", func() {
nested := newModel("nested", "file://nested.yaml", "", "")
nested.Candidates = []gallery.Candidate{{Model: "qwen3-8b-vllm-awq"}}
nested.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq"}}
models = append(models, nested)
base.Candidates = []gallery.Candidate{{Model: "nested"}}
_, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "")
base.Variants = []gallery.Variant{{Model: "nested"}}
_, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "")
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("nested"))
})
It("surfaces a bad pin", func() {
_, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "nope")
_, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "nope")
Expect(err).To(MatchError(gallery.ErrPinNotFound))
})
})
var _ = Describe("InstallModelFromGallery with candidate entries", func() {
var _ = Describe("InstallModelFromGallery with variant entries", func() {
var tempdir string
var galleries []config.Gallery
var systemState *system.SystemState
@@ -286,12 +301,12 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() {
return m
}
// withCandidates attaches upgrades to an otherwise ordinary entry. The
// floors are absolute rather than relative to the host: "0GiB" always
// matches and "10000GiB" never does, so these specs assert on resolution
// rather than on whatever VRAM the machine running them happens to have.
withCandidates := func(m gallery.GalleryModel, candidates ...gallery.Candidate) gallery.GalleryModel {
m.Candidates = candidates
// withVariants attaches alternative builds to an otherwise ordinary entry.
// The figures are absolute rather than relative to the host: "0GiB" always
// fits and "10000GiB" never does, so these specs assert on selection rather
// than on whatever memory the machine running them happens to have.
withVariants := func(m gallery.GalleryModel, variants ...gallery.Variant) gallery.GalleryModel {
m.Variants = variants
return m
}
@@ -311,7 +326,7 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() {
BeforeEach(func() {
var err error
tempdir, err = os.MkdirTemp("", "candidate-install")
tempdir, err = os.MkdirTemp("", "variant-install")
Expect(err).ToNot(HaveOccurred())
DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) })
@@ -319,42 +334,73 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() {
Expect(err).ToNot(HaveOccurred())
})
It("installs the entry's own payload when no candidate fits the host", func() {
It("installs the entry's own payload when no variant fits the host", func() {
newGallery(
withCandidates(entry("qwen3-8b-q4", "base-backend"),
gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "10000GiB"}),
withVariants(entry("qwen3-8b-q4", "base-backend"),
gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "10000GiB"}),
entry("qwen3-8b-q8", "upgrade-backend"),
)
// No machine clears a 10000GiB floor, so this asserts the base is the
// last resort and that missing every candidate is not an error.
// No machine clears 10000GiB, so this asserts the base is the last
// resort and that missing every variant is not an error.
Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed())
Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend"))
})
It("installs a fitting candidate's payload under the entry's own name", func() {
It("installs a fitting variant's payload under the entry's own name", func() {
newGallery(
withCandidates(entry("qwen3-8b-q4", "base-backend"),
gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}),
withVariants(entry("qwen3-8b-q4", "base-backend"),
gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}),
entry("qwen3-8b-q8", "upgrade-backend"),
)
// A 0GiB floor is met by every machine, so this asserts the upgrade wins
// over the base and lands under the base's name rather than its own.
Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed())
Expect(installedBackend("qwen3-8b-q4")).To(Equal("upgrade-backend"))
_, err := os.Stat(filepath.Join(tempdir, "qwen3-8b-q8.yaml"))
Expect(os.IsNotExist(err)).To(BeTrue(), "the upgrade must not be installed under its own name")
Expect(os.IsNotExist(err)).To(BeTrue(), "the variant must not be installed under its own name")
})
It("installs the largest fitting variant, not the first authored", func() {
// Authored smallest-first with all three within reach, so first-match
// would take the small one.
newGallery(
withVariants(entry("qwen3-8b-q4", "base-backend"),
gallery.Variant{Model: "qwen3-8b-small", MinMemory: "0GiB"},
gallery.Variant{Model: "qwen3-8b-large", MinMemory: "1KiB"},
),
entry("qwen3-8b-small", "small-backend"),
entry("qwen3-8b-large", "large-backend"),
)
Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed())
Expect(installedBackend("qwen3-8b-q4")).To(Equal("large-backend"))
})
It("never installs a variant whose backend cannot run on this host", func() {
if runtime.GOOS == "darwin" {
Skip("mlx is a compatible backend on darwin, so it proves nothing here")
}
// The mlx entry is the only variant and fits trivially, so the real
// SystemState.IsBackendCompatible rejecting mlx on a non-darwin host is
// the only thing that can send this to the base.
newGallery(
withVariants(entry("qwen3-8b-q4", "base-backend"),
gallery.Variant{Model: "qwen3-8b-mlx", MinMemory: "0GiB"}),
entry("qwen3-8b-mlx", "mlx"),
)
Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed())
Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend"))
})
It("round-trips the resolution record to disk under the entry's name", func() {
// The upgrade is described by url so its payload carries its own name.
// The variant is described by url so its payload carries its own name.
// Without the entry-name overlay the record persists as "qwen3-8b-q8",
// and the stable name the entry exists to provide is lost the moment
// anything reads the record back.
newGallery(
withCandidates(urlEntry("qwen3-8b-q4", "base-backend"),
gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}),
withVariants(urlEntry("qwen3-8b-q4", "base-backend"),
gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}),
urlEntry("qwen3-8b-q8", "upgrade-backend"),
)
@@ -372,8 +418,8 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() {
It("records a pin and honors it on a plain reinstall", func() {
newGallery(
withCandidates(entry("qwen3-8b-q4", "base-backend"),
gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}),
withVariants(entry("qwen3-8b-q4", "base-backend"),
gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}),
entry("qwen3-8b-q8", "upgrade-backend"),
)
@@ -385,7 +431,7 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() {
Expect(record.ResolvedVariant).To(Equal("qwen3-8b-q4"))
Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend"))
// No WithVariant this time: hardware resolution would take the upgrade,
// No WithVariant this time: hardware selection would take the upgrade,
// so only the recalled pin can keep this on the base payload.
Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed())
Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend"))
@@ -397,8 +443,8 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() {
It("honors a pin recorded under a custom install name", func() {
newGallery(
withCandidates(entry("qwen3-8b-q4", "base-backend"),
gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}),
withVariants(entry("qwen3-8b-q4", "base-backend"),
gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}),
entry("qwen3-8b-q8", "upgrade-backend"),
)
@@ -420,8 +466,8 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() {
})
It("writes each declared url only once into the persisted gallery file", func() {
base := withCandidates(entry("qwen3-8b-q4", "base-backend"),
gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"})
base := withVariants(entry("qwen3-8b-q4", "base-backend"),
gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"})
base.URLs = []string{"https://example.invalid/qwen3"}
newGallery(base, entry("qwen3-8b-q8", "upgrade-backend"))
@@ -434,12 +480,12 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() {
})
var _ = Describe("legacy client compatibility", func() {
It("keeps every entry that declares candidates installable by clients that ignore them", func() {
It("keeps every entry that declares variants installable by clients that ignore them", func() {
data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml"))
Expect(err).ToNot(HaveOccurred())
// Parse exactly as an older LocalAI release would: non-strictly, with
// no knowledge of the candidates key. Such a client installs whatever
// no knowledge of the variants key. Such a client installs whatever
// payload the entry carries directly, so the entry must carry one.
var legacy []struct {
Name string `yaml:"name"`
@@ -458,23 +504,23 @@ var _ = Describe("legacy client compatibility", func() {
legacyByName[e.Name] = i
}
withCandidates := 0
withVariants := 0
for _, e := range current {
if !e.HasCandidates() {
if !e.HasVariants() {
continue
}
withCandidates++
withVariants++
i, ok := legacyByName[e.Name]
Expect(ok).To(BeTrue(), "entry %q vanished under a legacy parse", e.Name)
old := legacy[i]
// An entry whose payload lived only in its candidates would install
// to nothing on every released LocalAI, which is precisely what
// making the entry itself the base candidate exists to prevent.
// An entry whose payload lived only in its variants would install to
// nothing on every released LocalAI, which is precisely what making
// the entry itself the last resort exists to prevent.
Expect(old.URL != "" || len(old.ConfigFile) > 0).To(BeTrue(),
"entry %q carries no payload of its own, so older clients would install nothing", e.Name)
}
Expect(withCandidates).To(BeNumerically(">", 0),
"expected at least one entry declaring candidates in the gallery index")
Expect(withVariants).To(BeNumerically(">", 0),
"expected at least one entry declaring variants in the gallery index")
})
})

281
core/gallery/variants_lint_test.go Normal file
View File

@@ -0,0 +1,281 @@
package gallery_test
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gopkg.in/yaml.v3"
"github.com/mudler/LocalAI/core/gallery"
)
// variantViolation is one invariant breach found by a lint helper. Helpers
// return every breach they find instead of stopping at the first, so a single
// run names them all rather than forcing a fix-one-rerun-repeat cycle.
type variantViolation struct {
Entry string
Variant string
Detail string
}
func (v variantViolation) String() string {
if v.Variant == "" {
return fmt.Sprintf("%s: %s", v.Entry, v.Detail)
}
return fmt.Sprintf("%s -> variant %q: %s", v.Entry, v.Variant, v.Detail)
}
func formatViolations(violations []variantViolation) string {
lines := make([]string, 0, len(violations))
for _, v := range violations {
lines = append(lines, " "+v.String())
}
return "\n" + strings.Join(lines, "\n")
}
func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel {
byName := make(map[string]gallery.GalleryModel, len(entries))
for _, e := range entries {
byName[e.Name] = e
}
return byName
}
// checkVariantReferences verifies every variant names an existing entry that
// declares no variants of its own. Selection is a single pass, so a nested
// reference would silently ignore the inner list.
//
// This is the one structural rule left. The rules about authored order, the
// hardware vocabulary and floor relationships stopped describing real hazards
// once selection became a filter plus a ranking: an author writes names, and
// there is no longer anything about hardware they can get wrong.
func checkVariantReferences(entries []gallery.GalleryModel) []variantViolation {
byName := indexEntriesByName(entries)
var violations []variantViolation
for _, e := range entries {
if !e.HasVariants() {
continue
}
for _, v := range e.Variants {
target, ok := byName[v.Model]
if !ok {
violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "references unknown model"})
continue
}
if target.HasVariants() {
violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "references an entry that declares variants of its own; nesting is not allowed"})
}
}
}
return violations
}
// checkVariantMemory verifies every declared memory figure parses, on the
// variants and on the entry itself.
//
// Absence is fine and deliberately not flagged: an unknown requirement is a
// legitimate state that selection handles by ranking the variant last. An
// UNPARSEABLE one is different, because it makes selection fail outright for
// the whole entry rather than degrade.
func checkVariantMemory(entries []gallery.GalleryModel) []variantViolation {
var violations []variantViolation
for _, e := range entries {
if !e.HasVariants() {
continue
}
base := gallery.Variant{Model: e.Name, MinMemory: e.MinMemory}
if _, _, err := base.EffectiveMinMemory(); err != nil {
violations = append(violations, variantViolation{Entry: e.Name, Detail: "has a bad min_memory: " + err.Error()})
}
for _, v := range e.Variants {
if _, _, err := v.EffectiveMinMemory(); err != nil {
violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "has a bad min_memory: " + err.Error()})
}
}
}
return violations
}
// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The
// index carries well over a thousand entries, so re-parsing it per spec is
// pure overhead.
var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) {
data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml"))
if err != nil {
return nil, err
}
var entries []gallery.GalleryModel
if err := yaml.Unmarshal(data, &entries); err != nil {
return nil, err
}
return entries, nil
})
func plainEntry(name, url string) gallery.GalleryModel {
e := gallery.GalleryModel{}
e.Name = name
e.URL = url
return e
}
func entryWithVariants(name, url, minMemory string, variants ...gallery.Variant) gallery.GalleryModel {
e := gallery.GalleryModel{Variants: variants, MinMemory: minMemory}
e.Name = name
e.URL = url
return e
}
// variantFixture builds an entry with variants plus the entries it references.
// Synthetic fixtures keep the invariant logic covered however few entries in
// gallery/index.yaml declare variants; without them every index-driven spec
// below is a no-op that passes while checking nothing.
func variantFixture(base gallery.GalleryModel, referenced ...gallery.GalleryModel) []gallery.GalleryModel {
return append([]gallery.GalleryModel{base}, referenced...)
}
var _ = Describe("gallery variant lint helpers", func() {
// An entry declares variants solely by carrying the key. GalleryBackend.IsMeta()
// has deliberately different semantics (it requires an EMPTY uri), so a
// well-meaning alignment of the two would make every helper skip every
// entry and pass silently. Assert the distinction directly.
It("treats an entry with variants as such even though it has a url", func() {
Expect(entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "big"}).HasVariants()).To(BeTrue())
Expect(plainEntry("big", "u://big").HasVariants()).To(BeFalse())
})
It("passes every invariant on a valid entry", func() {
entries := variantFixture(
entryWithVariants("base", "u://base", "4GiB",
gallery.Variant{Model: "big", MinMemory: "24GiB"},
gallery.Variant{Model: "mid", MinMemory: "12GiB"},
gallery.Variant{Model: "metal-big", MinMemory: "32GiB"},
gallery.Variant{Model: "small"},
),
plainEntry("big", "u://big"),
plainEntry("mid", "u://mid"),
plainEntry("metal-big", "u://metal-big"),
plainEntry("small", "u://small"),
)
Expect(checkVariantReferences(entries)).To(BeEmpty())
Expect(checkVariantMemory(entries)).To(BeEmpty())
})
It("passes every invariant on an entry that declares no memory at all", func() {
// Authoring is meant to be nothing but a list of names, so an entry and
// its variants carrying no figures must be entirely valid.
entries := variantFixture(
entryWithVariants("base", "u://base", "", gallery.Variant{Model: "big"}),
plainEntry("big", "u://big"),
)
Expect(checkVariantReferences(entries)).To(BeEmpty())
Expect(checkVariantMemory(entries)).To(BeEmpty())
})
Describe("checkVariantReferences", func() {
It("flags a variant naming an entry that does not exist", func() {
entries := variantFixture(
entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "ghost"}),
plainEntry("a", "u://a"),
)
violations := checkVariantReferences(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Variant).To(Equal("ghost"))
Expect(violations[0].Detail).To(ContainSubstring("unknown model"))
})
It("flags a variant naming an entry that declares variants itself", func() {
entries := variantFixture(
entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "nested"}),
entryWithVariants("nested", "u://nested", "4GiB", gallery.Variant{Model: "a"}),
plainEntry("a", "u://a"),
)
violations := checkVariantReferences(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Variant).To(Equal("nested"))
Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed"))
})
It("reports every breach in one pass rather than stopping at the first", func() {
entries := variantFixture(
entryWithVariants("base", "u://base", "2GiB",
gallery.Variant{Model: "ghost"},
gallery.Variant{Model: "phantom"},
),
)
Expect(checkVariantReferences(entries)).To(HaveLen(2))
})
})
Describe("checkVariantMemory", func() {
It("flags a variant whose memory figure cannot be parsed", func() {
entries := variantFixture(
entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "a", MinMemory: "lots"}),
plainEntry("a", "u://a"),
)
violations := checkVariantMemory(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Variant).To(Equal("a"))
Expect(violations[0].Detail).To(ContainSubstring("bad min_memory"))
})
It("flags an entry whose own memory figure cannot be parsed", func() {
entries := variantFixture(
entryWithVariants("base", "u://base", "eight gigs", gallery.Variant{Model: "a"}),
plainEntry("a", "u://a"),
)
violations := checkVariantMemory(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Entry).To(Equal("base"))
Expect(violations[0].Variant).To(BeEmpty())
Expect(violations[0].Detail).To(ContainSubstring("bad min_memory"))
})
It("flags an unparseable denormalized figure just as an authored one", func() {
// The nightly job writes inferred_min_memory, so a bug there breaks
// selection exactly as a bad hand-authored value does.
entries := variantFixture(
entryWithVariants("base", "u://base", "", gallery.Variant{Model: "a", InferredMinMemory: "8 gigabytes"}),
plainEntry("a", "u://a"),
)
violations := checkVariantMemory(entries)
Expect(violations).To(HaveLen(1))
Expect(violations[0].Variant).To(Equal("a"))
})
})
})
var _ = Describe("gallery/index.yaml variant invariants", Ordered, func() {
var entries []gallery.GalleryModel
BeforeAll(func() {
var err error
entries, err = loadGalleryIndex()
Expect(err).ToNot(HaveOccurred())
// A truncated or emptied index unmarshals cleanly and would make every
// spec below vacuously pass.
Expect(entries).ToNot(BeEmpty())
})
It("references only existing entries that declare no variants themselves", func() {
v := checkVariantReferences(entries)
Expect(v).To(BeEmpty(), formatViolations(v))
})
It("declares only parseable memory figures", func() {
v := checkVariantMemory(entries)
Expect(v).To(BeEmpty(), formatViolations(v))
})
})

26
core/schema/gallery-model.schema.json
View File

@@ -56,17 +56,13 @@
"additionalProperties": false
}
},
"capability": {
"min_memory": {
"type": "string",
"description": "Host capability this entry's own payload prefers, matched exactly and case-sensitively (for example metal, nvidia-cuda-12). Advisory: an unmet capability warns, it never blocks the install."
"description": "This entry's own memory requirement, for example 2GiB. One figure covers both VRAM and system RAM; the installer compares it against whichever the host will use. Advisory for the entry itself: falling short only warns."
},
"min_vram": {
"type": "string",
"description": "This entry's own VRAM floor, for example 2GiB. Positions the entry among its own candidates and must be strictly below every candidate's floor. Advisory: falling short only warns."
},
"candidates": {
"variants": {
"type": "array",
"description": "Ordered list of hardware-gated upgrades over this entry. The first candidate the host satisfies is installed, under this entry's own name; if none does, this entry installs itself. Clients that predate candidates support drop this key and install the entry unchanged.",
"description": "Other builds of the same model (other backends, other quantizations) the installer may pick instead of this entry's own payload. Order carries no meaning: the installer drops what this host cannot run or does not have the memory for, then takes the largest of what is left, installing it under this entry's own name. If nothing is left this entry installs itself. Clients that predate variants support drop this key and install the entry unchanged.",
"minItems": 1,
"items": {
"type": "object",
@@ -74,15 +70,11 @@
"properties": {
"model": {
"type": "string",
"description": "Name of a gallery entry that declares no candidates of its own"
"description": "Name of a gallery entry that declares no variants of its own"
},
"capability": {
"min_memory": {
"type": "string",
"description": "Host capability this candidate requires, matched exactly and case-sensitively (for example metal, nvidia-cuda-12). Absent matches any host."
},
"min_vram": {
"type": "string",
"description": "Authored VRAM floor, for example 20GiB. Authoritative: inference never overwrites it."
"description": "Authored memory requirement, for example 20GiB. Optional, and only needed to override an inferred estimate that is wrong. Authoritative: inference never overwrites it."
},
"backend": {
"type": "string",
@@ -92,9 +84,9 @@
"type": "string",
"description": "Denormalized by the nightly job for display. Never authored by hand."
},
"inferred_min_vram": {
"inferred_min_memory": {
"type": "string",
"description": "Denormalized by the nightly job as a fallback floor. Never authored by hand; min_vram wins."
"description": "Denormalized by the nightly job as a fallback requirement. Never authored by hand; min_memory wins."
}
},
"additionalProperties": false

13
gallery/index.yaml
View File

@@ -4296,13 +4296,14 @@
- math
last_checked: "2026-04-30"
# This entry installs the Q4_K_M build as-is, on any client and any host.
# min_vram positions it as its own last-resort candidate, and candidates
# lists the upgrades a bigger host gets instead. Clients that predate
# candidates support drop both keys and install this entry unchanged.
min_vram: 2GiB
candidates:
# variants lists other builds of the same model the installer may pick
# instead when this host can run them and has the memory for them. Clients
# that predate variants support drop both keys and install this entry
# unchanged.
min_memory: 2GiB
variants:
- model: nanbeige4.1-3b-q8
min_vram: 6GiB
min_memory: 6GiB
overrides:
parameters:
model: nanbeige4.1-3b-q4_k_m.gguf