diff --git a/.github/ci/gallery_denormalize.go b/.github/ci/gallery_denormalize.go index a650a6458..0ae528f47 100644 --- a/.github/ci/gallery_denormalize.go +++ b/.github/ci/gallery_denormalize.go @@ -1,5 +1,6 @@ -// Command gallery_denormalize fills the read-only denormalized fields on meta -// model entries: backend, quantization, and inferred_min_vram. +// Command gallery_denormalize fills the read-only denormalized fields on the +// candidates of gallery model entries: backend, quantization, and +// inferred_min_vram. // // It never modifies an authored min_vram. Authored values are authoritative, // because a human who measured a real load knows more than a pre-download @@ -50,7 +51,7 @@ func main() { failures := 0 for i := range entries { - if !entries[i].IsMeta() { + if !entries[i].HasCandidates() { continue } for j := range entries[i].Candidates { @@ -155,7 +156,7 @@ func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) e changed := false for i, entryNode := range root.Content { - if !entries[i].IsMeta() || entryNode.Kind != yaml.MappingNode { + if !entries[i].HasCandidates() || entryNode.Kind != yaml.MappingNode { continue } diff --git a/core/gallery/candidate.go b/core/gallery/candidate.go index cede3d934..9e5b34298 100644 --- a/core/gallery/candidate.go +++ b/core/gallery/candidate.go @@ -6,11 +6,13 @@ import ( "github.com/mudler/LocalAI/pkg/vram" ) -// Candidate is one option in a meta model entry's ordered candidate list. It -// references an existing concrete gallery entry by name and declares the -// conditions under which that entry is the right choice for the host. +// Candidate is one option in a gallery entry's ordered candidate list. It +// references an existing gallery entry by name and declares the conditions +// under which that entry is the right choice for the host. type Candidate struct { - // Model is the name of a concrete (non-meta) gallery entry. + // Model is the name of a gallery entry that declares no candidates of its + // own, or the name of the declaring entry itself when it stands as its own + // last-resort candidate. Model string `json:"model" yaml:"model"` // Capability, when set, must equal the host's reported capability // (e.g. "metal", "nvidia-cuda-12"). Empty matches any host. @@ -21,7 +23,7 @@ type Candidate struct { // The fields below are denormalized by the nightly job for display and // lint. They are never authored by hand and never affect what gets - // installed, because installation reads the concrete entry live. + // installed, because installation reads the referenced entry live. Backend string `json:"backend,omitempty" yaml:"backend,omitempty"` Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"` InferredMinVRAM string `json:"inferred_min_vram,omitempty" yaml:"inferred_min_vram,omitempty"` diff --git a/core/gallery/candidates_install_test.go b/core/gallery/candidates_install_test.go new file mode 100644 index 000000000..108f26e66 --- /dev/null +++ b/core/gallery/candidates_install_test.go @@ -0,0 +1,480 @@ +package gallery_test + +import ( + "context" + "os" + "path/filepath" + + "dario.cat/mergo" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" +) + +var _ = Describe("GalleryModel candidate declarations", func() { + It("declares no candidates when the key is absent", func() { + m := gallery.GalleryModel{} + m.Name = "plain" + Expect(m.HasCandidates()).To(BeFalse()) + }) + + It("declares candidates when the key is present", func() { + m := gallery.GalleryModel{Candidates: []gallery.Candidate{{Model: "x"}}} + Expect(m.HasCandidates()).To(BeTrue()) + }) + + It("parses an entry's own selection fields and its candidate list in order", func() { + var m gallery.GalleryModel + err := yaml.Unmarshal([]byte(` +name: qwen3-8b-gguf-q4 +url: "github:example/repo/qwen3-8b-gguf-q4.yaml@master" +min_vram: 6GiB +capability: default +candidates: + - model: qwen3-8b-vllm-awq + capability: nvidia + min_vram: 20GiB + - model: qwen3-8b-gguf-q8 + min_vram: 10GiB +`), &m) + Expect(err).ToNot(HaveOccurred()) + Expect(m.Name).To(Equal("qwen3-8b-gguf-q4")) + Expect(m.URL).To(Equal("github:example/repo/qwen3-8b-gguf-q4.yaml@master")) + Expect(m.MinVRAM).To(Equal("6GiB")) + Expect(m.Capability).To(Equal("default")) + Expect(m.HasCandidates()).To(BeTrue()) + Expect(m.Candidates).To(HaveLen(2)) + Expect(m.Candidates[0].Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(m.Candidates[0].Capability).To(Equal("nvidia")) + Expect(m.Candidates[0].MinVRAM).To(Equal("20GiB")) + Expect(m.Candidates[1].Model).To(Equal("qwen3-8b-gguf-q8")) + Expect(m.Candidates[1].Capability).To(BeEmpty()) + }) +}) + +var _ = Describe("ResolveVariant", func() { + gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } + + newModel := func(name, url, description, icon string) *gallery.GalleryModel { + m := &gallery.GalleryModel{} + m.Name = name + m.URL = url + m.Description = description + m.Icon = icon + return m + } + + var models []*gallery.GalleryModel + var base *gallery.GalleryModel + + BeforeEach(func() { + upgrade := newModel("qwen3-8b-vllm-awq", "file://vllm.yaml", "AWQ variant", "vllm.png") + // The base is an ordinary, complete entry that happens to declare an + // upgrade over itself, which is the whole point of the design. + base = newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "Qwen3 8B Q4", "qwen.png") + base.Tags = []string{"llm"} + base.MinVRAM = "6GiB" + base.Candidates = []gallery.Candidate{ + {Model: "qwen3-8b-vllm-awq", Capability: "nvidia", MinVRAM: "20GiB"}, + } + models = []*gallery.GalleryModel{upgrade, base} + }) + + It("installs a matching candidate's payload under the entry's name", func() { + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.Name).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("falls back to the entry's own payload when no candidate fits", func() { + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("installs the entry even when the host misses the entry's own floor", func() { + // There is nothing below the base, so refusing here would make an entry + // that every older client installs fine uninstallable on new ones. + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(1)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("strips the selection fields from the resolved entry", func() { + // A resolved entry is a concrete install target. Leaving the fields on + // it would let a second resolution pass fire on an already-resolved + // entry. + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(resolved.HasCandidates()).To(BeFalse()) + Expect(resolved.MinVRAM).To(BeEmpty()) + Expect(resolved.Capability).To(BeEmpty()) + }) + + It("presents the entry's metadata, not the candidate's", func() { + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(resolved.Description).To(Equal("Qwen3 8B Q4")) + Expect(resolved.Icon).To(Equal("qwen.png")) + Expect(resolved.Tags).To(ConsistOf("llm")) + }) + + It("honors a pin naming the entry itself", func() { + // The entry is the last element of its own candidate list, so its own + // name has to be a usable pin: it is how an operator declines an + // upgrade their hardware would otherwise take. + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "qwen3-8b-gguf-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.Name).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("honors a pin the hardware does not satisfy", func() { + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(2)}, "qwen3-8b-vllm-awq") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("detaches the resolved entry from the gallery's own maps and slices", func() { + models[0].Overrides = map[string]any{"f16": true} + models[0].AdditionalFiles = []gallery.File{{Filename: "a.bin"}} + + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // The install path merges the caller's request overrides into this map in + // place, so aliasing the gallery's map would write one caller's request + // into the shared catalog and leak it into every later install. + resolved.Overrides["f16"] = false + resolved.Overrides["threads"] = 4 + Expect(models[0].Overrides).To(HaveKeyWithValue("f16", true)) + Expect(models[0].Overrides).ToNot(HaveKey("threads")) + + resolved.AdditionalFiles[0].Filename = "mutated.bin" + Expect(models[0].AdditionalFiles[0].Filename).To(Equal("a.bin")) + + resolved.Tags = append(resolved.Tags, "extra") + Expect(base.Tags).To(ConsistOf("llm")) + }) + + It("detaches the resolved entry even when it resolves to the entry itself", func() { + // Resolving to the base returns a copy of the very entry the gallery + // holds, which is the case most likely to alias it. + base.Overrides = map[string]any{"parameters": map[string]any{"model": "q4.gguf"}} + + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).ToNot(HaveOccurred()) + + resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf" + Expect(base.Overrides["parameters"]).To(HaveKeyWithValue("model", "q4.gguf")) + }) + + It("detaches nested override maps from the gallery's own entry", func() { + models[0].Overrides = map[string]any{ + "parameters": map[string]any{"model": "real-variant.gguf"}, + "stopwords": []any{""}, + } + + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // Cloning only the top level would leave this inner map shared with the + // gallery entry, so writing through the resolved copy would rewrite the + // catalog's own payload. + resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf" + resolved.Overrides["stopwords"].([]any)[0] = "<|im_end|>" + + Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) + Expect(models[0].Overrides["stopwords"]).To(Equal([]any{""})) + }) + + It("does not write the caller's overrides back into the gallery entry", func() { + models[0].Overrides = map[string]any{"parameters": map[string]any{"model": "real-variant.gguf"}} + + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // This is exactly what the install path does with the caller's request + // overrides, and mergo recurses into nested maps and overwrites them in + // place. Asserting against the in-memory catalog is the only way to + // observe the leak: re-reading the gallery from disk re-unmarshals fresh + // maps and would pass whether or not the resolved entry was detached. + requestOverrides := map[string]any{"parameters": map[string]any{"model": "callers-choice.gguf"}} + Expect(mergo.Merge(&resolved.Overrides, requestOverrides, mergo.WithOverride)).To(Succeed()) + + Expect(resolved.Overrides["parameters"]).To(HaveKeyWithValue("model", "callers-choice.gguf")) + Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) + }) + + It("errors when a candidate references a missing entry", func() { + base.Candidates = []gallery.Candidate{{Model: "does-not-exist"}} + _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does-not-exist")) + }) + + It("refuses a candidate that declares candidates of its own", func() { + nested := newModel("nested", "file://nested.yaml", "", "") + nested.Candidates = []gallery.Candidate{{Model: "qwen3-8b-vllm-awq"}} + models = append(models, nested) + base.Candidates = []gallery.Candidate{{Model: "nested"}} + _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("nested")) + }) + + It("surfaces a bad pin", func() { + _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "nope") + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + }) +}) + +var _ = Describe("InstallModelFromGallery with candidate entries", func() { + var tempdir string + var galleries []config.Gallery + var systemState *system.SystemState + + // Every entry is described with an inline config_file rather than a URL so + // the whole install runs off the local filesystem with no network access. + newGallery := func(entries ...gallery.GalleryModel) { + out, err := yaml.Marshal(entries) + Expect(err).ToNot(HaveOccurred()) + galleryPath := filepath.Join(tempdir, "gallery.yaml") + Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed()) + + galleries = []config.Gallery{{Name: "test", URL: "file://" + galleryPath}} + } + + entry := func(name, backend string) gallery.GalleryModel { + m := gallery.GalleryModel{ConfigFile: map[string]any{"backend": backend}} + m.Name = name + m.Description = "entry " + name + return m + } + + // urlEntry describes an entry through a url rather than an inline + // config_file. The distinction matters for the recorded name: the url branch + // reads a name out of the fetched config, and that name is the referenced + // entry's own, so only the entry-name overlay can keep the record under the + // name the user asked for. The inline config_file branch seeds the name from + // the already-renamed resolved entry and so cannot observe the overlay. + urlEntry := func(name, backend string) gallery.GalleryModel { + payload := gallery.ModelConfig{ + Name: name, + Description: "entry " + name, + ConfigFile: "backend: " + backend + "\n", + } + out, err := yaml.Marshal(payload) + Expect(err).ToNot(HaveOccurred()) + payloadPath := filepath.Join(tempdir, "payload-"+name+".yaml") + Expect(os.WriteFile(payloadPath, out, 0600)).To(Succeed()) + + m := gallery.GalleryModel{} + m.Name = name + m.Description = "entry " + name + m.URL = "file://" + payloadPath + return m + } + + // withCandidates attaches upgrades to an otherwise ordinary entry. The + // floors are absolute rather than relative to the host: "0GiB" always + // matches and "10000GiB" never does, so these specs assert on resolution + // rather than on whatever VRAM the machine running them happens to have. + withCandidates := func(m gallery.GalleryModel, candidates ...gallery.Candidate) gallery.GalleryModel { + m.Candidates = candidates + return m + } + + install := func(name string, req gallery.GalleryModel, options ...gallery.InstallOption) error { + return gallery.InstallModelFromGallery( + context.TODO(), galleries, []config.Gallery{}, systemState, nil, + name, req, func(string, string, string, float64) {}, false, false, false, options...) + } + + installedBackend := func(name string) string { + dat, err := os.ReadFile(filepath.Join(tempdir, name+".yaml")) + Expect(err).ToNot(HaveOccurred()) + content := map[string]any{} + Expect(yaml.Unmarshal(dat, &content)).To(Succeed()) + return content["backend"].(string) + } + + BeforeEach(func() { + var err error + tempdir, err = os.MkdirTemp("", "candidate-install") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) }) + + systemState, err = system.GetSystemState(system.WithModelPath(tempdir)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("installs the entry's own payload when no candidate fits the host", func() { + newGallery( + withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "10000GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + // No machine clears a 10000GiB floor, so this asserts the base is the + // last resort and that missing every candidate is not an error. + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) + }) + + It("installs a fitting candidate's payload under the entry's own name", func() { + newGallery( + withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + // A 0GiB floor is met by every machine, so this asserts the upgrade wins + // over the base and lands under the base's name rather than its own. + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("upgrade-backend")) + _, err := os.Stat(filepath.Join(tempdir, "qwen3-8b-q8.yaml")) + Expect(os.IsNotExist(err)).To(BeTrue(), "the upgrade must not be installed under its own name") + }) + + It("round-trips the resolution record to disk under the entry's name", func() { + // The upgrade is described by url so its payload carries its own name. + // Without the entry-name overlay the record persists as "qwen3-8b-q8", + // and the stable name the entry exists to provide is lost the moment + // anything reads the record back. + newGallery( + withCandidates(urlEntry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + urlEntry("qwen3-8b-q8", "upgrade-backend"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.EntryName).To(Equal("qwen3-8b-q4")) + Expect(record.ResolvedVariant).To(Equal("qwen3-8b-q8")) + Expect(record.PinnedVariant).To(BeEmpty()) + Expect(record.Name).To(Equal("qwen3-8b-q4")) + Expect(record.Description).To(Equal("entry qwen3-8b-q4")) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("upgrade-backend")) + }) + + It("records a pin and honors it on a plain reinstall", func() { + newGallery( + withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-q4"))).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-q4")) + Expect(record.ResolvedVariant).To(Equal("qwen3-8b-q4")) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) + + // No WithVariant this time: hardware resolution would take the upgrade, + // so only the recalled pin can keep this on the base payload. + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) + + record, err = gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-q4")) + }) + + It("honors a pin recorded under a custom install name", func() { + newGallery( + withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + req := gallery.GalleryModel{} + req.Name = "prod-llm" + + Expect(install("qwen3-8b-q4", req, gallery.WithVariant("qwen3-8b-q4"))).To(Succeed()) + + // The pin is written under the installed name, so the recall must read it + // back under that name too and not under the gallery entry's name. + record, err := gallery.GetLocalModelConfiguration(tempdir, "prod-llm") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-q4")) + Expect(record.EntryName).To(Equal("qwen3-8b-q4")) + Expect(installedBackend("prod-llm")).To(Equal("base-backend")) + + Expect(install("qwen3-8b-q4", req)).To(Succeed()) + Expect(installedBackend("prod-llm")).To(Equal("base-backend")) + }) + + It("writes each declared url only once into the persisted gallery file", func() { + base := withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}) + base.URLs = []string{"https://example.invalid/qwen3"} + newGallery(base, entry("qwen3-8b-q8", "upgrade-backend")) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.URLs).To(ConsistOf("https://example.invalid/qwen3")) + }) +}) + +var _ = Describe("legacy client compatibility", func() { + It("keeps every entry that declares candidates installable by clients that ignore them", func() { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Parse exactly as an older LocalAI release would: non-strictly, with + // no knowledge of the candidates key. Such a client installs whatever + // payload the entry carries directly, so the entry must carry one. + var legacy []struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + ConfigFile map[string]any `yaml:"config_file"` + Files []gallery.File `yaml:"files"` + Overrides map[string]any `yaml:"overrides"` + } + Expect(yaml.Unmarshal(data, &legacy)).To(Succeed()) + + var current []gallery.GalleryModel + Expect(yaml.Unmarshal(data, ¤t)).To(Succeed()) + + legacyByName := map[string]int{} + for i, e := range legacy { + legacyByName[e.Name] = i + } + + withCandidates := 0 + for _, e := range current { + if !e.HasCandidates() { + continue + } + withCandidates++ + + i, ok := legacyByName[e.Name] + Expect(ok).To(BeTrue(), "entry %q vanished under a legacy parse", e.Name) + old := legacy[i] + // An entry whose payload lived only in its candidates would install + // to nothing on every released LocalAI, which is precisely what + // making the entry itself the base candidate exists to prevent. + Expect(old.URL != "" || len(old.ConfigFile) > 0).To(BeTrue(), + "entry %q carries no payload of its own, so older clients would install nothing", e.Name) + } + Expect(withCandidates).To(BeNumerically(">", 0), + "expected at least one entry declaring candidates in the gallery index") + }) +}) diff --git a/core/gallery/candidates_lint_test.go b/core/gallery/candidates_lint_test.go new file mode 100644 index 000000000..5ba9dd52d --- /dev/null +++ b/core/gallery/candidates_lint_test.go @@ -0,0 +1,544 @@ +package gallery_test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/gallery" +) + +// knownCapabilities mirrors the values SystemState.DetectedCapability() can +// actually report, which is what the candidate resolver compares against with +// a case-sensitive exact match. A capability outside this set can never match, +// so a typo would silently make a candidate unreachable on every host. +// +// Note "cpu" is deliberately absent: it exists only as a fallback key inside +// SystemState.Capability(capMap) for meta backends, and is never a value +// getSystemCapabilities() returns. A CPU-only host reports "default". +var knownCapabilities = map[string]bool{ + "default": true, "metal": true, "darwin-x86": true, + "nvidia": true, "nvidia-cuda-12": true, "nvidia-cuda-13": true, + "nvidia-l4t": true, "nvidia-l4t-cuda-12": true, "nvidia-l4t-cuda-13": true, + "intel": true, "amd": true, "vulkan": true, +} + +// candidateViolation is one invariant breach found by a lint helper. Helpers +// return every breach they find instead of stopping at the first, so a single +// run names them all rather than forcing a fix-one-rerun-repeat cycle. +type candidateViolation struct { + Entry string + Candidate string + Detail string +} + +func (v candidateViolation) String() string { + if v.Candidate == "" { + return fmt.Sprintf("%s: %s", v.Entry, v.Detail) + } + return fmt.Sprintf("%s -> candidate %q: %s", v.Entry, v.Candidate, v.Detail) +} + +func formatViolations(violations []candidateViolation) string { + lines := make([]string, 0, len(violations)) + for _, v := range violations { + lines = append(lines, " "+v.String()) + } + return "\n" + strings.Join(lines, "\n") +} + +func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel { + byName := make(map[string]gallery.GalleryModel, len(entries)) + for _, e := range entries { + byName[e.Name] = e + } + return byName +} + +// baseCandidate mirrors the implicit last-resort candidate the resolver +// synthesizes from the entry itself, so lint measures the same floor the +// installer will. +func baseCandidate(e gallery.GalleryModel) gallery.Candidate { + return gallery.Candidate{Model: e.Name, Capability: e.Capability, MinVRAM: e.MinVRAM} +} + +// checkCandidateReferences verifies every candidate names an existing entry +// that declares no candidates of its own. Resolution is a single pass, so a +// nested reference would silently ignore the inner list. +func checkCandidateReferences(entries []gallery.GalleryModel) []candidateViolation { + byName := indexEntriesByName(entries) + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + for _, c := range e.Candidates { + target, ok := byName[c.Model] + if !ok { + violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "references unknown model"}) + continue + } + if target.HasCandidates() { + violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "references an entry that declares candidates of its own; nesting is not allowed"}) + } + } + } + return violations +} + +// checkCandidateFloors verifies every declared candidate carries a parseable +// VRAM floor. A candidate is an UPGRADE over the entry, so it must say what it +// costs; a floorless one would capture every host and make the entry's own +// payload unreachable. +func checkCandidateFloors(entries []gallery.GalleryModel) []candidateViolation { + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + for _, c := range e.Candidates { + _, declared, err := c.EffectiveMinVRAM() + switch { + case err != nil: + violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "has a bad min_vram: " + err.Error()}) + case !declared: + violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "needs a min_vram; the nightly job should have inferred one"}) + } + } + } + return violations +} + +// checkBaseFloor verifies the entry's own floor sits strictly below every +// candidate's. The entry is the last element of its own candidate list, so a +// base floor at or above a candidate's makes that candidate dead: any host +// clearing it already cleared the base, and first-match never gets that far. +func checkBaseFloor(entries []gallery.GalleryModel) []candidateViolation { + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + base := baseCandidate(e) + baseFloor, _, err := base.EffectiveMinVRAM() + if err != nil { + violations = append(violations, candidateViolation{Entry: e.Name, Detail: "has a bad min_vram: " + err.Error()}) + continue + } + for _, c := range e.Candidates { + floor, declared, cerr := c.EffectiveMinVRAM() + if cerr != nil || !declared { + // checkCandidateFloors owns reporting those. + continue + } + if baseFloor >= floor { + violations = append(violations, candidateViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: fmt.Sprintf("sits at or below the entry's own %d byte floor, so it can never be reached", + baseFloor), + }) + } + } + } + return violations +} + +// checkCandidateCapabilities verifies entries and candidates only name +// capabilities the system can actually report, since the resolver compares +// them exactly. +func checkCandidateCapabilities(entries []gallery.GalleryModel) []candidateViolation { + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + if e.Capability != "" && !knownCapabilities[e.Capability] { + violations = append(violations, candidateViolation{ + Entry: e.Name, + Detail: fmt.Sprintf("uses unknown capability %q", e.Capability), + }) + } + for _, c := range e.Candidates { + if c.Capability == "" { + continue + } + if !knownCapabilities[c.Capability] { + violations = append(violations, candidateViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: fmt.Sprintf("uses unknown capability %q", c.Capability), + }) + } + } + } + return violations +} + +// checkCandidateOrdering verifies no candidate is shadowed by an earlier one. +// Selection is first-match over the authored order, so a candidate is dead +// whenever an earlier one matches every host this one would. +// +// Two shapes cause that. Within a single capability group the groups are +// mutually exclusive, so only a floor rising above an earlier floor is +// unreachable. A candidate with an EMPTY capability matches every host and so +// dominates ACROSS groups: every later candidate whose floor is at or above +// the running minimum unconditional floor is dead, whatever capability it asks +// for. Tracking that running minimum subsumes the same-group check for the +// empty capability. +func checkCandidateOrdering(entries []gallery.GalleryModel) []candidateViolation { + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + previous := map[string]uint64{} + var unconditionalFloor uint64 + haveUnconditional := false + + for _, c := range e.Candidates { + floor, declared, err := c.EffectiveMinVRAM() + if err != nil || !declared { + // Parse errors and absent floors belong to checkCandidateFloors. + continue + } + + switch { + case haveUnconditional && floor >= unconditionalFloor: + violations = append(violations, candidateViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: fmt.Sprintf("is shadowed by an earlier candidate that matches any host at a %d byte floor, so it can never be reached", + unconditionalFloor), + }) + case c.Capability != "": + if prior, seen := previous[c.Capability]; seen && floor > prior { + violations = append(violations, candidateViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: "raises the VRAM floor after a lower one in the same capability group, so it can never be reached", + }) + } + } + + if c.Capability == "" && (!haveUnconditional || floor < unconditionalFloor) { + unconditionalFloor = floor + haveUnconditional = true + } + previous[c.Capability] = floor + } + } + return violations +} + +// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The +// index carries well over a thousand entries, so re-parsing it per spec is +// pure overhead. +var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) + if err != nil { + return nil, err + } + var entries []gallery.GalleryModel + if err := yaml.Unmarshal(data, &entries); err != nil { + return nil, err + } + return entries, nil +}) + +func plainEntry(name, url string) gallery.GalleryModel { + e := gallery.GalleryModel{} + e.Name = name + e.URL = url + return e +} + +func entryWithCandidates(name, url, minVRAM string, candidates ...gallery.Candidate) gallery.GalleryModel { + e := gallery.GalleryModel{Candidates: candidates, MinVRAM: minVRAM} + e.Name = name + e.URL = url + return e +} + +// candidateFixture builds an entry with candidates plus the entries it +// references. Synthetic fixtures keep the invariant logic covered however few +// entries in gallery/index.yaml declare candidates; without them every +// index-driven spec below is a no-op that passes while checking nothing. +func candidateFixture(base gallery.GalleryModel, referenced ...gallery.GalleryModel) []gallery.GalleryModel { + return append([]gallery.GalleryModel{base}, referenced...) +} + +var _ = Describe("gallery candidate lint helpers", func() { + // An entry declares candidates solely by carrying the key. GalleryBackend.IsMeta() + // has deliberately different semantics (it requires an EMPTY uri), so a + // well-meaning alignment of the two would make every helper skip every + // entry and pass silently. Assert the distinction directly. + It("treats an entry with candidates as such even though it has a url", func() { + Expect(entryWithCandidates("base", "u://base", "2GiB", gallery.Candidate{Model: "big"}).HasCandidates()).To(BeTrue()) + Expect(plainEntry("big", "u://big").HasCandidates()).To(BeFalse()) + }) + + It("passes every invariant on a valid entry", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "4GiB", + gallery.Candidate{Model: "big", Capability: "nvidia", MinVRAM: "24GiB"}, + gallery.Candidate{Model: "mid", Capability: "nvidia", MinVRAM: "12GiB"}, + gallery.Candidate{Model: "metal-big", Capability: "metal", MinVRAM: "32GiB"}, + gallery.Candidate{Model: "small", MinVRAM: "8GiB"}, + ), + plainEntry("big", "u://big"), + plainEntry("mid", "u://mid"), + plainEntry("metal-big", "u://metal-big"), + plainEntry("small", "u://small"), + ) + + Expect(checkCandidateReferences(entries)).To(BeEmpty()) + Expect(checkCandidateFloors(entries)).To(BeEmpty()) + Expect(checkBaseFloor(entries)).To(BeEmpty()) + Expect(checkCandidateCapabilities(entries)).To(BeEmpty()) + Expect(checkCandidateOrdering(entries)).To(BeEmpty()) + }) + + It("passes every invariant on an entry that declares no floor of its own", func() { + // A floorless entry is the weakest possible base, below every + // candidate, which is exactly the position the base should hold. + entries := candidateFixture( + entryWithCandidates("base", "u://base", "", + gallery.Candidate{Model: "big", MinVRAM: "24GiB"}, + ), + plainEntry("big", "u://big"), + ) + + Expect(checkCandidateFloors(entries)).To(BeEmpty()) + Expect(checkBaseFloor(entries)).To(BeEmpty()) + Expect(checkCandidateOrdering(entries)).To(BeEmpty()) + }) + + Describe("checkBaseFloor", func() { + It("flags a candidate whose floor sits below the entry's own", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "20GiB", + gallery.Candidate{Model: "big", MinVRAM: "8GiB"}, + ), + plainEntry("big", "u://big"), + ) + + violations := checkBaseFloor(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("big")) + Expect(violations[0].Detail).To(ContainSubstring("at or below the entry's own")) + }) + + It("flags a candidate whose floor merely equals the entry's own", func() { + // Equal floors make the candidate unreachable just as surely: every + // host clearing it clears the base, and the base is checked first + // only in the sense that first-match never reaches this candidate. + entries := candidateFixture( + entryWithCandidates("base", "u://base", "8GiB", + gallery.Candidate{Model: "big", MinVRAM: "8GiB"}, + ), + plainEntry("big", "u://big"), + ) + + violations := checkBaseFloor(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("big")) + }) + + It("flags an entry whose own min_vram cannot be parsed", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "eight gigs", + gallery.Candidate{Model: "big", MinVRAM: "24GiB"}, + ), + plainEntry("big", "u://big"), + ) + + violations := checkBaseFloor(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Entry).To(Equal("base")) + Expect(violations[0].Candidate).To(BeEmpty()) + Expect(violations[0].Detail).To(ContainSubstring("bad min_vram")) + }) + }) + + Describe("checkCandidateOrdering", func() { + It("flags a candidate shadowed by an earlier unconditional candidate", func() { + // "a" matches any host at 8GiB, so the nvidia candidate at 20GiB is + // dead: every nvidia host clearing 20GiB already cleared 8GiB. + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, + ), + plainEntry("a", "u://a"), plainEntry("b", "u://b"), + ) + + violations := checkCandidateOrdering(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("b")) + Expect(violations[0].Detail).To(ContainSubstring("shadowed by an earlier candidate that matches any host")) + }) + + It("flags a floor inversion inside one capability group", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, + ), + plainEntry("a", "u://a"), plainEntry("b", "u://b"), + ) + + violations := checkCandidateOrdering(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("b")) + Expect(violations[0].Detail).To(ContainSubstring("same capability group")) + }) + + It("keeps distinct capability groups independent", func() { + // A high metal floor after a low nvidia floor is fine: no host + // reports both capabilities. + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "metal", MinVRAM: "32GiB"}, + ), + plainEntry("a", "u://a"), plainEntry("b", "u://b"), + ) + + Expect(checkCandidateOrdering(entries)).To(BeEmpty()) + }) + }) + + Describe("checkCandidateFloors", func() { + It("flags a candidate with no floor", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", Capability: "nvidia"}, + ), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateFloors(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring("needs a min_vram")) + }) + + It("flags a candidate whose floor cannot be parsed", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", MinVRAM: "lots"}, + ), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateFloors(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring("bad min_vram")) + }) + }) + + Describe("checkCandidateReferences", func() { + It("flags a candidate naming an entry that does not exist", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "ghost", MinVRAM: "20GiB"}, + ), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateReferences(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("ghost")) + Expect(violations[0].Detail).To(ContainSubstring("unknown model")) + }) + + It("flags a candidate naming an entry that declares candidates itself", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "nested", MinVRAM: "20GiB"}, + ), + entryWithCandidates("nested", "u://nested", "4GiB", gallery.Candidate{Model: "a", MinVRAM: "30GiB"}), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateReferences(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("nested")) + Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed")) + }) + }) + + Describe("checkCandidateCapabilities", func() { + It("flags a capability the system can never report", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", Capability: "cuda", MinVRAM: "20GiB"}, + ), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateCapabilities(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "cuda"`)) + }) + + It("flags an unknown capability on the entry itself", func() { + entry := entryWithCandidates("base", "u://base", "2GiB", gallery.Candidate{Model: "a", MinVRAM: "20GiB"}) + entry.Capability = "NVIDIA" + entries := candidateFixture(entry, plainEntry("a", "u://a")) + + violations := checkCandidateCapabilities(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Entry).To(Equal("base")) + Expect(violations[0].Candidate).To(BeEmpty()) + Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "NVIDIA"`)) + }) + }) +}) + +var _ = Describe("gallery/index.yaml candidate invariants", Ordered, func() { + var entries []gallery.GalleryModel + + BeforeAll(func() { + var err error + entries, err = loadGalleryIndex() + Expect(err).ToNot(HaveOccurred()) + // A truncated or emptied index unmarshals cleanly and would make every + // spec below vacuously pass. + Expect(entries).ToNot(BeEmpty()) + }) + + It("references only existing entries that declare no candidates themselves", func() { + v := checkCandidateReferences(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("gives every candidate a VRAM floor", func() { + v := checkCandidateFloors(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("keeps every entry's own floor below its candidates'", func() { + v := checkBaseFloor(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("uses only capabilities the system can report", func() { + v := checkCandidateCapabilities(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("orders candidates so no candidate is shadowed by an earlier one", func() { + v := checkCandidateOrdering(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) +}) diff --git a/core/gallery/meta_install_test.go b/core/gallery/meta_install_test.go deleted file mode 100644 index ee57162a7..000000000 --- a/core/gallery/meta_install_test.go +++ /dev/null @@ -1,407 +0,0 @@ -package gallery_test - -import ( - "context" - "os" - "path/filepath" - - "dario.cat/mergo" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" - - "github.com/mudler/LocalAI/core/config" - "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/pkg/system" -) - -var _ = Describe("GalleryModel meta entries", func() { - It("is not meta when it has no candidates", func() { - m := gallery.GalleryModel{} - m.Name = "plain" - Expect(m.IsMeta()).To(BeFalse()) - }) - - It("is meta when it has candidates", func() { - m := gallery.GalleryModel{Candidates: []gallery.Candidate{{Model: "x"}}} - Expect(m.IsMeta()).To(BeTrue()) - }) - - It("parses a candidate list from gallery YAML in order", func() { - var m gallery.GalleryModel - err := yaml.Unmarshal([]byte(` -name: qwen3-8b -url: "github:example/repo/qwen3-8b-gguf-q4.yaml@master" -candidates: - - model: qwen3-8b-vllm-awq - capability: nvidia - min_vram: 20GiB - - model: qwen3-8b-gguf-q4 -`), &m) - Expect(err).ToNot(HaveOccurred()) - Expect(m.IsMeta()).To(BeTrue()) - Expect(m.Candidates).To(HaveLen(2)) - Expect(m.Candidates[0].Model).To(Equal("qwen3-8b-vllm-awq")) - Expect(m.Candidates[0].Capability).To(Equal("nvidia")) - Expect(m.Candidates[0].MinVRAM).To(Equal("20GiB")) - Expect(m.Candidates[1].Model).To(Equal("qwen3-8b-gguf-q4")) - Expect(m.Candidates[1].Capability).To(BeEmpty()) - }) -}) - -var _ = Describe("ResolveMetaModel", func() { - gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } - - newModel := func(name, url, description, icon string) *gallery.GalleryModel { - m := &gallery.GalleryModel{} - m.Name = name - m.URL = url - m.Description = description - m.Icon = icon - return m - } - - var models []*gallery.GalleryModel - var meta *gallery.GalleryModel - - BeforeEach(func() { - concreteVLLM := newModel("qwen3-8b-vllm-awq", "file://vllm.yaml", "AWQ variant", "vllm.png") - concreteGGUF := newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "GGUF variant", "gguf.png") - meta = newModel("qwen3-8b", "file://gguf.yaml", "Qwen3 8B", "qwen.png") - meta.Tags = []string{"llm"} - meta.Candidates = []gallery.Candidate{ - {Model: "qwen3-8b-vllm-awq", Capability: "nvidia", MinVRAM: "20GiB"}, - {Model: "qwen3-8b-gguf-q4"}, - } - models = []*gallery.GalleryModel{concreteVLLM, concreteGGUF, meta} - }) - - It("installs the concrete payload under the meta's name", func() { - resolved, candidate, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) - Expect(resolved.Name).To(Equal("qwen3-8b")) - Expect(resolved.URL).To(Equal("file://vllm.yaml")) - }) - - It("presents the meta's metadata, not the variant's", func() { - resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - Expect(resolved.Description).To(Equal("Qwen3 8B")) - Expect(resolved.Icon).To(Equal("qwen.png")) - Expect(resolved.Tags).To(ConsistOf("llm")) - }) - - It("keeps the name stable when a variant is pinned", func() { - resolved, candidate, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "qwen3-8b-gguf-q4") - Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) - Expect(resolved.Name).To(Equal("qwen3-8b")) - Expect(resolved.URL).To(Equal("file://gguf.yaml")) - }) - - It("detaches the resolved entry from the gallery's own maps and slices", func() { - models[0].Overrides = map[string]any{"f16": true} - models[0].AdditionalFiles = []gallery.File{{Filename: "a.bin"}} - - resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - - // The install path merges the caller's request overrides into this map in - // place, so aliasing the gallery's map would write one caller's request - // into the shared catalog and leak it into every later install. - resolved.Overrides["f16"] = false - resolved.Overrides["threads"] = 4 - Expect(models[0].Overrides).To(HaveKeyWithValue("f16", true)) - Expect(models[0].Overrides).ToNot(HaveKey("threads")) - - resolved.AdditionalFiles[0].Filename = "mutated.bin" - Expect(models[0].AdditionalFiles[0].Filename).To(Equal("a.bin")) - - resolved.Tags = append(resolved.Tags, "extra") - Expect(meta.Tags).To(ConsistOf("llm")) - }) - - It("detaches nested override maps from the gallery's own entry", func() { - models[0].Overrides = map[string]any{ - "parameters": map[string]any{"model": "real-variant.gguf"}, - "stopwords": []any{""}, - } - - resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - - // Cloning only the top level would leave this inner map shared with the - // gallery entry, so writing through the resolved copy would rewrite the - // catalog's own payload. - resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf" - resolved.Overrides["stopwords"].([]any)[0] = "<|im_end|>" - - Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) - Expect(models[0].Overrides["stopwords"]).To(Equal([]any{""})) - }) - - It("does not write the caller's overrides back into the gallery entry", func() { - models[0].Overrides = map[string]any{"parameters": map[string]any{"model": "real-variant.gguf"}} - - resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - - // This is exactly what the install path does with the caller's request - // overrides, and mergo recurses into nested maps and overwrites them in - // place. Asserting against the in-memory catalog is the only way to - // observe the leak: re-reading the gallery from disk re-unmarshals fresh - // maps and would pass whether or not the resolved entry was detached. - requestOverrides := map[string]any{"parameters": map[string]any{"model": "callers-choice.gguf"}} - Expect(mergo.Merge(&resolved.Overrides, requestOverrides, mergo.WithOverride)).To(Succeed()) - - Expect(resolved.Overrides["parameters"]).To(HaveKeyWithValue("model", "callers-choice.gguf")) - Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) - }) - - It("errors when a candidate references a missing entry", func() { - meta.Candidates = []gallery.Candidate{{Model: "does-not-exist"}} - _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("does-not-exist")) - }) - - It("refuses a candidate that is itself a meta entry", func() { - nested := newModel("nested", "", "", "") - nested.Candidates = []gallery.Candidate{{Model: "qwen3-8b-gguf-q4"}} - models = append(models, nested) - meta.Candidates = []gallery.Candidate{{Model: "nested"}} - _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("nested")) - }) - - It("surfaces a bad pin", func() { - _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "nope") - Expect(err).To(MatchError(gallery.ErrPinNotFound)) - }) -}) - -var _ = Describe("InstallModelFromGallery with meta entries", func() { - var tempdir string - var galleries []config.Gallery - var systemState *system.SystemState - - // The variants are described with an inline config_file rather than a URL so - // the whole install runs off the local filesystem with no network access. - // The meta entry keeps a url as well, because a real meta entry carries one - // as a fallback for older LocalAI releases that do not understand candidates, - // and installing that fallback instead of a variant is exactly the regression - // these specs guard against. - newGallery := func(meta gallery.GalleryModel, variants ...gallery.GalleryModel) { - fallback := gallery.ModelConfig{ - Name: "legacy-fallback", - Description: "legacy fallback payload", - ConfigFile: "backend: fallback-backend\n", - } - fallbackYAML, err := yaml.Marshal(fallback) - Expect(err).ToNot(HaveOccurred()) - fallbackPath := filepath.Join(tempdir, "fallback.yaml") - Expect(os.WriteFile(fallbackPath, fallbackYAML, 0600)).To(Succeed()) - - meta.URL = "file://" + fallbackPath - entries := append([]gallery.GalleryModel{meta}, variants...) - - out, err := yaml.Marshal(entries) - Expect(err).ToNot(HaveOccurred()) - galleryPath := filepath.Join(tempdir, "gallery.yaml") - Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed()) - - galleries = []config.Gallery{{Name: "test", URL: "file://" + galleryPath}} - } - - variant := func(name, backend string) gallery.GalleryModel { - m := gallery.GalleryModel{ConfigFile: map[string]any{"backend": backend}} - m.Name = name - m.Description = "variant " + name - return m - } - - // urlVariant describes a variant through a url rather than an inline - // config_file. The distinction matters for the recorded name: the url branch - // reads a name out of the fetched config, and that name is the variant's own, - // so only the meta-name overlay can keep the record under the meta's name. - // The inline config_file branch seeds the name from the already-renamed - // resolved entry and so cannot observe the overlay at all. - urlVariant := func(name, backend string) gallery.GalleryModel { - payload := gallery.ModelConfig{ - Name: name, - Description: "variant " + name, - ConfigFile: "backend: " + backend + "\n", - } - out, err := yaml.Marshal(payload) - Expect(err).ToNot(HaveOccurred()) - payloadPath := filepath.Join(tempdir, "payload-"+name+".yaml") - Expect(os.WriteFile(payloadPath, out, 0600)).To(Succeed()) - - m := gallery.GalleryModel{} - m.Name = name - m.Description = "variant " + name - m.URL = "file://" + payloadPath - return m - } - - metaEntry := func(name string, candidates ...string) gallery.GalleryModel { - m := gallery.GalleryModel{} - m.Name = name - m.Description = "the meta entry" - m.Icon = "meta.png" - for _, c := range candidates { - m.Candidates = append(m.Candidates, gallery.Candidate{Model: c}) - } - return m - } - - install := func(name string, req gallery.GalleryModel, options ...gallery.InstallOption) error { - return gallery.InstallModelFromGallery( - context.TODO(), galleries, []config.Gallery{}, systemState, nil, - name, req, func(string, string, string, float64) {}, false, false, false, options...) - } - - installedBackend := func(name string) string { - dat, err := os.ReadFile(filepath.Join(tempdir, name+".yaml")) - Expect(err).ToNot(HaveOccurred()) - content := map[string]any{} - Expect(yaml.Unmarshal(dat, &content)).To(Succeed()) - return content["backend"].(string) - } - - BeforeEach(func() { - var err error - tempdir, err = os.MkdirTemp("", "meta-install") - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) }) - - systemState, err = system.GetSystemState(system.WithModelPath(tempdir)) - Expect(err).ToNot(HaveOccurred()) - - newGallery( - metaEntry("qwen3-8b", "qwen3-8b-variant-a", "qwen3-8b-variant-b"), - variant("qwen3-8b-variant-a", "variant-a-backend"), - variant("qwen3-8b-variant-b", "variant-b-backend"), - ) - }) - - It("installs the resolved variant's payload, not the meta's url fallback", func() { - Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) - - // If meta-ness stopped winning over the url, this would be - // "fallback-backend" and every meta entry would silently install the - // legacy payload instead of a hardware-appropriate variant. - Expect(installedBackend("qwen3-8b")).To(Equal("variant-a-backend")) - }) - - It("round-trips the resolution record to disk under the meta's name", func() { - // The variant is described by url so its payload carries its own name. - // Without the meta-name overlay the record persists as - // "qwen3-8b-variant-a", and the stable name a meta entry exists to - // provide is lost the moment anything reads the record back. - newGallery( - metaEntry("qwen3-8b", "qwen3-8b-variant-a", "qwen3-8b-variant-b"), - urlVariant("qwen3-8b-variant-a", "variant-a-backend"), - urlVariant("qwen3-8b-variant-b", "variant-b-backend"), - ) - - Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) - - record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") - Expect(err).ToNot(HaveOccurred()) - Expect(record.MetaName).To(Equal("qwen3-8b")) - Expect(record.ResolvedVariant).To(Equal("qwen3-8b-variant-a")) - Expect(record.PinnedVariant).To(BeEmpty()) - Expect(record.Name).To(Equal("qwen3-8b")) - Expect(record.Description).To(Equal("the meta entry")) - Expect(installedBackend("qwen3-8b")).To(Equal("variant-a-backend")) - }) - - It("records a pin and honors it on a plain reinstall", func() { - Expect(install("qwen3-8b", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-variant-b"))).To(Succeed()) - - record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") - Expect(err).ToNot(HaveOccurred()) - Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) - Expect(record.ResolvedVariant).To(Equal("qwen3-8b-variant-b")) - Expect(installedBackend("qwen3-8b")).To(Equal("variant-b-backend")) - - // No WithVariant this time: hardware resolution would pick variant-a, so - // only the recalled pin can keep this on variant-b. - Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) - Expect(installedBackend("qwen3-8b")).To(Equal("variant-b-backend")) - - record, err = gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") - Expect(err).ToNot(HaveOccurred()) - Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) - }) - - It("honors a pin recorded under a custom install name", func() { - req := gallery.GalleryModel{} - req.Name = "prod-llm" - - Expect(install("qwen3-8b", req, gallery.WithVariant("qwen3-8b-variant-b"))).To(Succeed()) - - // The pin is written under the installed name, so the recall must read it - // back under that name too and not under the gallery entry's name. - record, err := gallery.GetLocalModelConfiguration(tempdir, "prod-llm") - Expect(err).ToNot(HaveOccurred()) - Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) - Expect(record.MetaName).To(Equal("qwen3-8b")) - Expect(installedBackend("prod-llm")).To(Equal("variant-b-backend")) - - Expect(install("qwen3-8b", req)).To(Succeed()) - Expect(installedBackend("prod-llm")).To(Equal("variant-b-backend")) - }) - - It("writes each declared url only once into the persisted gallery file", func() { - meta := metaEntry("qwen3-8b", "qwen3-8b-variant-a") - meta.URLs = []string{"https://example.invalid/qwen3"} - newGallery(meta, variant("qwen3-8b-variant-a", "variant-a-backend")) - - Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) - - record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") - Expect(err).ToNot(HaveOccurred()) - Expect(record.URLs).To(ConsistOf("https://example.invalid/qwen3")) - }) -}) - -var _ = Describe("legacy client compatibility", func() { - It("exposes a url on every meta entry to clients that ignore candidates", func() { - data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) - Expect(err).ToNot(HaveOccurred()) - - // Parse exactly as an older LocalAI release would: non-strictly, with - // no knowledge of the candidates key. - var legacy []struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - } - Expect(yaml.Unmarshal(data, &legacy)).To(Succeed()) - - var current []gallery.GalleryModel - Expect(yaml.Unmarshal(data, ¤t)).To(Succeed()) - - urlByName := map[string]string{} - for _, e := range legacy { - urlByName[e.Name] = e.URL - } - - metaCount := 0 - for _, e := range current { - if !e.IsMeta() { - continue - } - metaCount++ - // Without a url an old client lists the entry and installs an - // empty model, because it silently drops candidates. - Expect(urlByName[e.Name]).ToNot(BeEmpty(), - "meta entry %q is invisible payload-wise to older clients", e.Name) - } - Expect(metaCount).To(BeNumerically(">", 0), - "expected at least one meta entry in the gallery index") - }) -}) diff --git a/core/gallery/meta_lint_test.go b/core/gallery/meta_lint_test.go deleted file mode 100644 index d964f1457..000000000 --- a/core/gallery/meta_lint_test.go +++ /dev/null @@ -1,510 +0,0 @@ -package gallery_test - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "sync" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" - - "github.com/mudler/LocalAI/core/gallery" -) - -// knownCapabilities mirrors the values SystemState.DetectedCapability() can -// actually report, which is what the candidate resolver compares against with -// a case-sensitive exact match. A capability outside this set can never match, -// so a typo would silently make a candidate unreachable on every host. -// -// Note "cpu" is deliberately absent: it exists only as a fallback key inside -// SystemState.Capability(capMap) for meta backends, and is never a value -// getSystemCapabilities() returns. A CPU-only host reports "default". -var knownCapabilities = map[string]bool{ - "default": true, "metal": true, "darwin-x86": true, - "nvidia": true, "nvidia-cuda-12": true, "nvidia-cuda-13": true, - "nvidia-l4t": true, "nvidia-l4t-cuda-12": true, "nvidia-l4t-cuda-13": true, - "intel": true, "amd": true, "vulkan": true, -} - -// metaViolation is one invariant breach found by a lint helper. Helpers return -// every breach they find instead of stopping at the first, so a single run -// names them all rather than forcing a fix-one-rerun-repeat cycle. -type metaViolation struct { - Entry string - Candidate string - Detail string -} - -func (v metaViolation) String() string { - if v.Candidate == "" { - return fmt.Sprintf("%s: %s", v.Entry, v.Detail) - } - return fmt.Sprintf("%s -> candidate %q: %s", v.Entry, v.Candidate, v.Detail) -} - -func formatViolations(violations []metaViolation) string { - lines := make([]string, 0, len(violations)) - for _, v := range violations { - lines = append(lines, " "+v.String()) - } - return "\n" + strings.Join(lines, "\n") -} - -func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel { - byName := make(map[string]gallery.GalleryModel, len(entries)) - for _, e := range entries { - byName[e.Name] = e - } - return byName -} - -// checkMetaFallbackURL verifies that released LocalAI versions, which ignore -// the candidates key entirely, still install something sensible: the meta -// entry needs a url, and it must be the final candidate's url so old and new -// clients agree on the least demanding option. -func checkMetaFallbackURL(entries []gallery.GalleryModel) []metaViolation { - byName := indexEntriesByName(entries) - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - if e.URL == "" { - violations = append(violations, metaViolation{Entry: e.Name, Detail: "needs a url fallback for older clients"}) - } - if len(e.ConfigFile) > 0 { - violations = append(violations, metaViolation{Entry: e.Name, Detail: "must not carry an inline config_file"}) - } - if len(e.AdditionalFiles) > 0 { - violations = append(violations, metaViolation{Entry: e.Name, Detail: "must not carry files"}) - } - - last := e.Candidates[len(e.Candidates)-1] - fallback, ok := byName[last.Model] - if !ok { - // checkMetaReferences owns reporting the dangling name; there is - // nothing to compare the url against here. - continue - } - if e.URL != fallback.URL { - violations = append(violations, metaViolation{ - Entry: e.Name, - Candidate: last.Model, - Detail: "meta url must equal its final candidate url so old and new clients agree", - }) - } - } - return violations -} - -// checkMetaReferences verifies every candidate names an existing, concrete -// entry. Resolution is a single pass, so a nested meta reference would install -// an entry that carries no files. -func checkMetaReferences(entries []gallery.GalleryModel) []metaViolation { - byName := indexEntriesByName(entries) - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - for _, c := range e.Candidates { - target, ok := byName[c.Model] - if !ok { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "references unknown model"}) - continue - } - if target.IsMeta() { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "references a meta entry; nesting is not allowed"}) - } - } - } - return violations -} - -// checkMetaConstraints verifies that only the final candidate is an -// unconstrained last resort. An earlier candidate without a floor would -// capture every host and make everything after it dead. -func checkMetaConstraints(entries []gallery.GalleryModel) []metaViolation { - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - for i, c := range e.Candidates { - _, declared, err := c.EffectiveMinVRAM() - if err != nil { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "has a bad min_vram: " + err.Error()}) - continue - } - - if i == len(e.Candidates)-1 { - if declared { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "final candidate must be an unconstrained last resort"}) - } - if c.Capability != "" { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "final candidate must not require a capability"}) - } - continue - } - if !declared { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "needs a min_vram; the nightly job should have inferred one"}) - } - } - } - return violations -} - -// checkMetaCapabilities verifies candidates only name capabilities the system -// can actually report, since the resolver compares them exactly. -func checkMetaCapabilities(entries []gallery.GalleryModel) []metaViolation { - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - for _, c := range e.Candidates { - if c.Capability == "" { - continue - } - if !knownCapabilities[c.Capability] { - violations = append(violations, metaViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: fmt.Sprintf("uses unknown capability %q", c.Capability), - }) - } - } - } - return violations -} - -// checkMetaOrdering verifies no candidate is shadowed by an earlier one. -// Selection is first-match over the authored order, so a candidate is dead -// whenever an earlier one matches every host this one would. -// -// Two shapes cause that. Within a single capability group the groups are -// mutually exclusive, so only a floor rising above an earlier floor is -// unreachable. A candidate with an EMPTY capability matches every host and so -// dominates ACROSS groups: every later candidate whose floor is at or above -// the running minimum unconditional floor is dead, whatever capability it asks -// for. Tracking that running minimum subsumes the same-group check for the -// empty capability. -func checkMetaOrdering(entries []gallery.GalleryModel) []metaViolation { - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - previous := map[string]uint64{} - var unconditionalFloor uint64 - haveUnconditional := false - - for _, c := range e.Candidates { - floor, declared, err := c.EffectiveMinVRAM() - if err != nil || !declared { - // Parse errors and absent floors belong to checkMetaConstraints. - continue - } - - switch { - case haveUnconditional && floor >= unconditionalFloor: - violations = append(violations, metaViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: fmt.Sprintf("is shadowed by an earlier candidate that matches any host at a %d byte floor, so it can never be reached", - unconditionalFloor), - }) - case c.Capability != "": - if prior, seen := previous[c.Capability]; seen && floor > prior { - violations = append(violations, metaViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: "raises the VRAM floor after a lower one in the same capability group, so it can never be reached", - }) - } - } - - if c.Capability == "" && (!haveUnconditional || floor < unconditionalFloor) { - unconditionalFloor = floor - haveUnconditional = true - } - previous[c.Capability] = floor - } - } - return violations -} - -// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The -// index carries well over a thousand entries, so re-parsing it per spec is -// pure overhead. -var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) { - data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) - if err != nil { - return nil, err - } - var entries []gallery.GalleryModel - if err := yaml.Unmarshal(data, &entries); err != nil { - return nil, err - } - return entries, nil -}) - -func concreteEntry(name, url string) gallery.GalleryModel { - e := gallery.GalleryModel{} - e.Name = name - e.URL = url - return e -} - -func metaEntry(name, url string, candidates ...gallery.Candidate) gallery.GalleryModel { - e := gallery.GalleryModel{Candidates: candidates} - e.Name = name - e.URL = url - return e -} - -// metaFixture builds a meta entry plus the concrete entries it references. -// Synthetic fixtures keep the invariant logic covered while gallery/index.yaml -// still holds zero meta entries; without them every index-driven spec below is -// a no-op that passes while checking nothing. -func metaFixture(meta gallery.GalleryModel, concrete ...gallery.GalleryModel) []gallery.GalleryModel { - return append([]gallery.GalleryModel{meta}, concrete...) -} - -var _ = Describe("meta entry lint helpers", func() { - // A model entry is meta solely by carrying candidates. GalleryBackend.IsMeta() - // has deliberately opposite semantics (it requires an EMPTY uri), so a - // well-meaning alignment of the two would make every helper skip every - // entry and pass silently. Assert the distinction directly. - It("treats an entry with candidates as meta even though it has a url", func() { - Expect(metaEntry("meta", "u://big", gallery.Candidate{Model: "big"}).IsMeta()).To(BeTrue()) - Expect(concreteEntry("big", "u://big").IsMeta()).To(BeFalse()) - }) - - It("passes every invariant on a valid meta entry", func() { - entries := metaFixture( - metaEntry("meta", "u://small", - gallery.Candidate{Model: "big", Capability: "nvidia", MinVRAM: "24GiB"}, - gallery.Candidate{Model: "mid", Capability: "nvidia", MinVRAM: "12GiB"}, - gallery.Candidate{Model: "metal-big", Capability: "metal", MinVRAM: "32GiB"}, - gallery.Candidate{Model: "small"}, - ), - concreteEntry("big", "u://big"), - concreteEntry("mid", "u://mid"), - concreteEntry("metal-big", "u://metal-big"), - concreteEntry("small", "u://small"), - ) - - Expect(checkMetaFallbackURL(entries)).To(BeEmpty()) - Expect(checkMetaReferences(entries)).To(BeEmpty()) - Expect(checkMetaConstraints(entries)).To(BeEmpty()) - Expect(checkMetaCapabilities(entries)).To(BeEmpty()) - Expect(checkMetaOrdering(entries)).To(BeEmpty()) - }) - - Describe("checkMetaOrdering", func() { - It("flags a candidate shadowed by an earlier unconditional candidate", func() { - // "a" matches any host at 8GiB, so the nvidia candidate at 20GiB is - // dead: every nvidia host clearing 20GiB already cleared 8GiB. - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaOrdering(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("b")) - Expect(violations[0].Detail).To(ContainSubstring("shadowed by an earlier candidate that matches any host")) - }) - - It("flags a floor inversion inside one capability group", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaOrdering(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("b")) - Expect(violations[0].Detail).To(ContainSubstring("same capability group")) - }) - - It("keeps distinct capability groups independent", func() { - // A high metal floor after a low nvidia floor is fine: no host - // reports both capabilities. - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "metal", MinVRAM: "32GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), - ) - - Expect(checkMetaOrdering(entries)).To(BeEmpty()) - }) - }) - - Describe("checkMetaConstraints", func() { - It("flags a non-final candidate with no floor", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", Capability: "nvidia"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaConstraints(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("a")) - Expect(violations[0].Detail).To(ContainSubstring("needs a min_vram")) - }) - - It("flags a final candidate that declares a floor", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c", MinVRAM: "8GiB"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaConstraints(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("c")) - Expect(violations[0].Detail).To(ContainSubstring("unconstrained last resort")) - }) - - It("flags a final candidate that requires a capability", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c", Capability: "nvidia"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaConstraints(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("c")) - Expect(violations[0].Detail).To(ContainSubstring("must not require a capability")) - }) - }) - - Describe("checkMetaReferences", func() { - It("flags a candidate naming an entry that does not exist", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "ghost", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("c", "u://c"), - ) - - violations := checkMetaReferences(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("ghost")) - Expect(violations[0].Detail).To(ContainSubstring("unknown model")) - }) - - It("flags a candidate naming another meta entry", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "nested", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - metaEntry("nested", "u://c", gallery.Candidate{Model: "c"}), - concreteEntry("c", "u://c"), - ) - - violations := checkMetaReferences(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("nested")) - Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed")) - }) - }) - - Describe("checkMetaFallbackURL", func() { - It("flags a meta url that differs from its final candidate url", func() { - entries := metaFixture( - metaEntry("meta", "u://wrong", - gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaFallbackURL(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("c")) - Expect(violations[0].Detail).To(ContainSubstring("must equal its final candidate url")) - }) - }) - - Describe("checkMetaCapabilities", func() { - It("flags a capability the system can never report", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", Capability: "cuda", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaCapabilities(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("a")) - Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "cuda"`)) - }) - }) -}) - -var _ = Describe("gallery/index.yaml meta entry invariants", Ordered, func() { - var entries []gallery.GalleryModel - - BeforeAll(func() { - var err error - entries, err = loadGalleryIndex() - Expect(err).ToNot(HaveOccurred()) - // A truncated or emptied index unmarshals cleanly and would make every - // spec below vacuously pass. - Expect(entries).ToNot(BeEmpty()) - }) - - It("gives every meta entry a legacy url and no inline payload", func() { - v := checkMetaFallbackURL(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("references only existing, non-meta entries", func() { - v := checkMetaReferences(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("constrains every candidate except an unconstrained last resort", func() { - v := checkMetaConstraints(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("uses only capabilities the system can report", func() { - v := checkMetaCapabilities(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("orders candidates so no candidate is shadowed by an earlier one", func() { - v := checkMetaOrdering(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) -}) diff --git a/core/gallery/model_artifacts.go b/core/gallery/model_artifacts.go index 01de02d03..d323293e6 100644 --- a/core/gallery/model_artifacts.go +++ b/core/gallery/model_artifacts.go @@ -32,8 +32,10 @@ func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption { } } -// WithVariant pins a meta model entry to a specific candidate by name, -// bypassing hardware-based resolution. Ignored for non-meta entries. +// WithVariant pins a gallery entry to a specific candidate by name, bypassing +// hardware-based resolution. The entry's own name is a valid pin, since the +// entry is itself the last-resort candidate. Ignored for entries that declare +// no candidates. func WithVariant(variant string) InstallOption { return func(options *installOptions) { options.variant = variant diff --git a/core/gallery/models.go b/core/gallery/models.go index 252dc539c..917e38618 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -61,10 +61,10 @@ type ModelConfig struct { Files []File `yaml:"files"` PromptTemplates []PromptTemplate `yaml:"prompt_templates"` - // The fields below record how a meta entry was resolved, so a reinstall - // or upgrade can honor the same pin and so operators can see which - // variant a stable model name is actually backed by. - MetaName string `yaml:"meta_name,omitempty"` + // The fields below record how an entry carrying candidates was resolved, + // so a reinstall or upgrade can honor the same pin and so operators can + // see which variant a stable model name is actually backed by. + EntryName string `yaml:"entry_name,omitempty"` ResolvedVariant string `yaml:"resolved_variant,omitempty"` PinnedVariant string `yaml:"pinned_variant,omitempty"` } @@ -80,26 +80,48 @@ type PromptTemplate struct { Content string `yaml:"content"` } -// ResolveMetaModel turns a meta gallery entry into the concrete entry that -// should be installed on this host, returning it renamed to the meta's name -// and carrying the meta's presentation metadata. +// effectiveCandidates returns the candidate list resolution actually runs over: +// the declared upgrades followed by the entry itself. +// +// The entry is appended rather than special-cased so there is exactly one +// selection path. Being last makes it the last resort, which is what keeps the +// entry always installable: the list can never be exhausted without reaching it. +func effectiveCandidates(entry *GalleryModel) []Candidate { + candidates := make([]Candidate, 0, len(entry.Candidates)+1) + candidates = append(candidates, entry.Candidates...) + return append(candidates, Candidate{ + Model: entry.Name, + Capability: entry.Capability, + MinVRAM: entry.MinVRAM, + }) +} + +// ResolveVariant picks the gallery entry to install for a host, from the +// upgrades an entry declares plus the entry itself, and returns it renamed to +// the entry's name and carrying the entry's presentation metadata. // // Why the metadata split: the payload (url, config_file, files, overrides) -// must come from the variant because that is what actually gets downloaded, -// while the presentation (name, description, icon, tags) must come from the -// meta so the installed model presents as the model rather than the variant. -func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Candidate, error) { - candidate, err := ResolveCandidate(meta.Candidates, env, pin) - if err != nil { - return nil, Candidate{}, fmt.Errorf("resolving variant for model %q: %w", meta.Name, err) - } +// must come from the chosen variant because that is what actually gets +// downloaded, while the presentation (name, description, icon, tags) must come +// from the entry the user asked for, so the installed model presents as that +// model rather than as one of its variants. +// +// The base entry always resolves. When even its own predicates fall short this +// warns and installs it regardless, because the entry is a complete entry that +// every older LocalAI release installs unconditionally; refusing it here would +// make the gallery behave worse the newer the client is. +func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Candidate, error) { + candidates := effectiveCandidates(entry) + base := candidates[len(candidates)-1] - concrete := FindGalleryElement(models, candidate.Model) - if concrete == nil { - return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", meta.Name, candidate.Model) - } - if concrete.IsMeta() { - return nil, Candidate{}, fmt.Errorf("model %q references variant %q which is itself a meta entry; meta entries may not nest", meta.Name, candidate.Model) + candidate, err := ResolveCandidate(candidates, env, pin) + if err != nil { + if !errors.Is(err, ErrNoCandidateMatch) { + return nil, Candidate{}, fmt.Errorf("resolving variant for model %q: %w", entry.Name, err) + } + xlog.Warn("This system does not meet the requirements this model declares for itself; installing it anyway because it is the last resort", + "model", entry.Name, "capability", env.Capability, "available_vram", env.VRAM, "reason", err) + candidate = base } // A pin is an operator override and deliberately bypasses the hardware @@ -108,19 +130,41 @@ func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv // It is warned about only once the pin is known to name a real, installable // entry, otherwise a pin naming a listed-but-nonexistent variant would warn // about VRAM and then fail for an entirely unrelated reason. - if pin != "" { + warnPin := func() { + if pin == "" { + return + } if floor, declared, verr := candidate.EffectiveMinVRAM(); verr == nil && declared && env.VRAM < floor { xlog.Warn("Pinned model variant declares more VRAM than this system reports; installing anyway because the pin overrides hardware resolution", - "model", meta.Name, "variant", candidate.Model, "required_vram", floor, "available_vram", env.VRAM) + "model", entry.Name, "variant", candidate.Model, "required_vram", floor, "available_vram", env.VRAM) } } - resolved := *concrete - resolved.Name = meta.Name - resolved.Description = meta.Description - resolved.Icon = meta.Icon - resolved.License = meta.License + // Resolving to the base means installing the entry's own payload, which is + // the entry itself; there is no second entry to look up. + source := entry + if candidate.Model != entry.Name { + source = FindGalleryElement(models, candidate.Model) + if source == nil { + return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", entry.Name, candidate.Model) + } + if source.HasCandidates() { + return nil, Candidate{}, fmt.Errorf("model %q references variant %q which declares candidates of its own; resolution is a single pass, so those would be silently ignored", entry.Name, candidate.Model) + } + } + warnPin() + + resolved := *source + resolved.Name = entry.Name + resolved.Description = entry.Description + resolved.Icon = entry.Icon + resolved.License = entry.License + // The resolved entry is a concrete install target, so it must not carry the + // selection fields any more; leaving them would let a second pass resolve + // the already-resolved entry all over again. resolved.Candidates = nil + resolved.Capability = "" + resolved.MinVRAM = "" // The struct copy above is shallow, so every reference-typed field still // aliases the gallery's own entries. The install path mutates Overrides in @@ -135,11 +179,11 @@ func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv // maps and overwrites them in place, so a top-level clone would still hand // the caller the gallery's own inner maps. The slices below hold value types, // so cloning them once fully detaches them. - resolved.Overrides = deepCopyStringMap(concrete.Overrides) - resolved.ConfigFile = deepCopyStringMap(concrete.ConfigFile) - resolved.AdditionalFiles = slices.Clone(concrete.AdditionalFiles) - resolved.URLs = slices.Clone(meta.URLs) - resolved.Tags = slices.Clone(meta.Tags) + resolved.Overrides = deepCopyStringMap(source.Overrides) + resolved.ConfigFile = deepCopyStringMap(source.ConfigFile) + resolved.AdditionalFiles = slices.Clone(source.AdditionalFiles) + resolved.URLs = slices.Clone(entry.URLs) + resolved.Tags = slices.Clone(entry.Tags) return &resolved, candidate, nil } @@ -228,13 +272,13 @@ func InstallModelFromGallery( } if record != nil { - config.MetaName = record.MetaName + config.EntryName = record.EntryName config.ResolvedVariant = record.ResolvedVariant config.PinnedVariant = record.PinnedVariant // The variant's own name would otherwise be persisted here, which - // contradicts the whole point of a meta entry: the model is known by - // the meta's stable name regardless of which variant backs it. - config.Name = record.MetaName + // contradicts the whole point of candidates: the model is known by + // the entry's stable name regardless of which variant backs it. + config.Name = record.EntryName } installName := model.Name @@ -281,10 +325,10 @@ func InstallModelFromGallery( return fmt.Errorf("no model found with name %q", name) } - // Meta-ness is checked before anything looks at the URL: a meta entry also - // carries a url as a fallback for older LocalAI releases that do not - // understand candidates, so carrying both is normal and meta wins here. - if !model.IsMeta() { + // An entry without candidates is installed directly. An entry with them is + // still installable as-is; resolution below only decides whether one of its + // declared upgrades fits this host better. + if !model.HasCandidates() { return applyModel(model, nil) } @@ -296,7 +340,7 @@ func InstallModelFromGallery( // The record is keyed by the name the model was installed under, not by the // gallery entry name: applyModel writes it to ._gallery_.yaml, // where installName is req.Name whenever the caller supplied one. Reading it - // back under the meta's own name would miss the record for every custom-named + // back under the entry's own name would miss the record for every custom-named // install and silently re-resolve a deliberately pinned model onto a // different variant, possibly swapping its backend. if pin == "" { @@ -314,17 +358,17 @@ func InstallModelFromGallery( VRAM: systemState.VRAM, } - resolved, candidate, err := ResolveMetaModel(models, model, env, pin) + resolved, candidate, err := ResolveVariant(models, model, env, pin) if err != nil { return err } - xlog.Info("Resolved meta model to variant", + xlog.Info("Resolved model to variant", "model", model.Name, "variant", candidate.Model, "capability", env.Capability, "vram", env.VRAM, "pinned", pin != "") return applyModel(resolved, &ModelConfig{ - MetaName: model.Name, + EntryName: model.Name, ResolvedVariant: candidate.Model, PinnedVariant: pin, }) diff --git a/core/gallery/models_types.go b/core/gallery/models_types.go index b50dfbcc1..3b570d8cc 100644 --- a/core/gallery/models_types.go +++ b/core/gallery/models_types.go @@ -15,10 +15,19 @@ type GalleryModel struct { ConfigFile map[string]any `json:"config_file,omitempty" yaml:"config_file,omitempty"` // Overrides are used to override the configuration of the model located at URL Overrides map[string]any `json:"overrides,omitempty" yaml:"overrides,omitempty"` - // Candidates, when non-empty, makes this a meta entry: an ordered list of - // concrete gallery entries, the first of which the host satisfies is what - // gets installed. Mirrors GalleryBackend's capabilities map one level up. + // Candidates is an optional ordered list of hardware-gated UPGRADES over + // this entry. The entry itself is always the last-resort candidate, so an + // entry carrying candidates stays a complete, installable entry and older + // LocalAI releases, which drop this key, install it exactly as before. Candidates []Candidate `json:"candidates,omitempty" yaml:"candidates,omitempty"` + // Capability, when set, is the host capability this entry's own payload + // prefers. It is advisory for the base entry: an unmet capability warns + // rather than refusing, because the base always installs. + Capability string `json:"capability,omitempty" yaml:"capability,omitempty"` + // MinVRAM is this entry's own VRAM floor, in the same form as a + // candidate's (e.g. "2GiB"). It positions the entry within its own + // candidate list and is likewise advisory: falling short only warns. + MinVRAM string `json:"min_vram,omitempty" yaml:"min_vram,omitempty"` } func (m *GalleryModel) GetInstalled() bool { @@ -49,9 +58,11 @@ func (m GalleryModel) ID() string { return fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name) } -// IsMeta reports whether this entry resolves to one of several concrete -// entries based on host hardware, rather than describing files directly. -func (m GalleryModel) IsMeta() bool { +// HasCandidates reports whether this entry declares hardware-gated upgrades +// over its own payload. It says nothing about the entry being installable: +// an entry with candidates is a complete entry that can always be installed +// as-is. +func (m GalleryModel) HasCandidates() bool { return len(m.Candidates) > 0 } diff --git a/core/schema/gallery-model.schema.json b/core/schema/gallery-model.schema.json index d43b42e07..d72004fbd 100644 --- a/core/schema/gallery-model.schema.json +++ b/core/schema/gallery-model.schema.json @@ -56,9 +56,17 @@ "additionalProperties": false } }, + "capability": { + "type": "string", + "description": "Host capability this entry's own payload prefers, matched exactly and case-sensitively (for example metal, nvidia-cuda-12). Advisory: an unmet capability warns, it never blocks the install." + }, + "min_vram": { + "type": "string", + "description": "This entry's own VRAM floor, for example 2GiB. Positions the entry among its own candidates and must be strictly below every candidate's floor. Advisory: falling short only warns." + }, "candidates": { "type": "array", - "description": "Ordered variant list of a meta entry. The first candidate the host satisfies is installed, under the meta entry's own name. An entry carrying candidates must not also carry files or config_file.", + "description": "Ordered list of hardware-gated upgrades over this entry. The first candidate the host satisfies is installed, under this entry's own name; if none does, this entry installs itself. Clients that predate candidates support drop this key and install the entry unchanged.", "minItems": 1, "items": { "type": "object", @@ -66,7 +74,7 @@ "properties": { "model": { "type": "string", - "description": "Name of a concrete (non-meta) gallery entry" + "description": "Name of a gallery entry that declares no candidates of its own" }, "capability": { "type": "string", diff --git a/gallery/index.yaml b/gallery/index.yaml index c449afa6e..57ebc707c 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -4234,35 +4234,6 @@ - filename: llama-cpp/models/Qwen_Qwen3-Next-80B-A3B-Thinking-Q4_K_M.gguf sha256: 83481c75cc6c0837ba9afa52b59b4cd3f85f55dd7aa6c60e27230ff329c81367 uri: https://huggingface.co/bartowski/Qwen_Qwen3-Next-80B-A3B-Thinking-GGUF/resolve/main/Qwen_Qwen3-Next-80B-A3B-Thinking-Q4_K_M.gguf -- name: nanbeige4.1-3b - description: | - Nanbeige4.1-3B is built upon Nanbeige4-3B-Base and represents an enhanced iteration of our previous reasoning model, Nanbeige4-3B-Thinking-2511, achieved through further post-training optimization with supervised fine-tuning (SFT) and reinforcement learning (RL). As a highly competitive open-source model at a small parameter scale, Nanbeige4.1-3B illustrates that compact models can simultaneously achieve robust reasoning, preference alignment, and effective agentic behaviors. - - Automatically resolves to the best variant for your hardware: the Q8_0 build where there is enough VRAM for it, the Q4_K_M build otherwise. - license: apache-2.0 - icon: https://cdn-avatars.huggingface.co/v1/production/uploads/646f0d118ff94af23bc44aab/GXHCollpMRgvYqUXQ2BQ7.png - urls: - - https://huggingface.co/Nanbeige/Nanbeige4.1-3B - tags: - - nanbeige - - 3b - - llm - - gguf - - quantized - - chat - - reasoning - - agent - - multilingual - - instruction-tuned - # url is the fallback for LocalAI releases that predate candidates support: - # they drop the candidates key silently and would otherwise install nothing. - # Lint requires it to equal the final candidate's url exactly, so old and new - # clients agree on the least demanding option. - url: github:mudler/LocalAI/gallery/nanbeige4.1.yaml@master - candidates: - - model: nanbeige4.1-3b-q8 - min_vram: 6GiB - - model: nanbeige4.1-3b-q4 - name: nanbeige4.1-3b-q8 url: github:mudler/LocalAI/gallery/nanbeige4.1.yaml@master urls: @@ -4324,6 +4295,14 @@ - code - math last_checked: "2026-04-30" + # This entry installs the Q4_K_M build as-is, on any client and any host. + # min_vram positions it as its own last-resort candidate, and candidates + # lists the upgrades a bigger host gets instead. Clients that predate + # candidates support drop both keys and install this entry unchanged. + min_vram: 2GiB + candidates: + - model: nanbeige4.1-3b-q8 + min_vram: 6GiB overrides: parameters: model: nanbeige4.1-3b-q4_k_m.gguf