diff --git a/core/gallery/empty_base_install_test.go b/core/gallery/empty_base_install_test.go new file mode 100644 index 000000000..6b104e233 --- /dev/null +++ b/core/gallery/empty_base_install_test.go @@ -0,0 +1,243 @@ +package gallery_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" +) + +// An entry declaring neither url: nor config_file: is installed on an empty +// base config, with overrides: and files: supplying everything. These specs +// cover that path, the payload rule that still rejects an entry with nothing to +// install, and the two older paths, which must be untouched. +// +// Nothing here reaches the network. The whole point of the empty-base path is +// that it fetches nothing, and a spec that quietly went to GitHub would be +// asserting the opposite of the change. +var _ = Describe("InstallModelFromGallery with an empty base config", func() { + var tempdir string + var galleries []config.Gallery + var systemState *system.SystemState + // The gallery listing is cached on the name and URL pair, so every spec + // needs a gallery of its own or it reads the previous spec's catalog. + galleryRevision := 0 + + newGallery := func(entries ...gallery.GalleryModel) { + out, err := yaml.Marshal(entries) + Expect(err).ToNot(HaveOccurred()) + name := fmt.Sprintf("empty-base-%d", galleryRevision) + galleryRevision++ + galleryPath := filepath.Join(tempdir, name+".yaml") + Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed()) + galleries = []config.Gallery{{Name: name, URL: "file://" + galleryPath}} + } + + install := func(name string, req gallery.GalleryModel, options ...gallery.InstallOption) error { + return gallery.InstallModelFromGallery( + context.TODO(), galleries, []config.Gallery{}, systemState, nil, + name, req, func(string, string, string, float64) {}, false, false, false, options...) + } + + installedConfig := func(name string) map[string]any { + dat, err := os.ReadFile(filepath.Join(tempdir, name+".yaml")) + Expect(err).ToNot(HaveOccurred()) + content := map[string]any{} + Expect(yaml.Unmarshal(dat, &content)).To(Succeed()) + return content + } + + // localWeights lets a spec carry a files: list without leaving the + // filesystem. The downloader treats an already-present destination with no + // declared sha256 as fetched and skips it, so seeding the file is what keeps + // these specs off the network. The URI is still authored, because the + // installer walks the list either way and a missing one would not exercise + // the same code. + localWeights := func(name string) gallery.File { + Expect(os.WriteFile(filepath.Join(tempdir, name), []byte("weights for "+name), 0600)).To(Succeed()) + return gallery.File{Filename: name, URI: "https://example.com/" + name} + } + + BeforeEach(func() { + var err error + tempdir, err = os.MkdirTemp("", "empty-base-install") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) }) + + systemState, err = system.GetSystemState(system.WithModelPath(tempdir)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("installs an entry that declares only overrides and files", func() { + e := gallery.GalleryModel{Overrides: map[string]any{ + "backend": "ds4", + "parameters": map[string]any{"model": "weights.gguf"}, + }} + e.Name = "overrides-only" + e.Description = "an entry with no base config" + e.AdditionalFiles = []gallery.File{localWeights("weights.gguf")} + newGallery(e) + + Expect(install("overrides-only", gallery.GalleryModel{})).To(Succeed()) + + cfg := installedConfig("overrides-only") + Expect(cfg["backend"]).To(Equal("ds4")) + Expect(cfg["parameters"]).To(HaveKeyWithValue("model", "weights.gguf")) + // The name comes from the install, never from a base config, which is + // what the several hundred virtual.yaml entries were getting wrong: they + // inherited the stub's name. + Expect(cfg["name"]).To(Equal("overrides-only")) + Expect(filepath.Join(tempdir, "weights.gguf")).To(BeAnExistingFile()) + }) + + It("installs an entry that declares only files", func() { + e := gallery.GalleryModel{} + e.Name = "files-only" + e.AdditionalFiles = []gallery.File{localWeights("plain.gguf")} + newGallery(e) + + Expect(install("files-only", gallery.GalleryModel{})).To(Succeed()) + Expect(filepath.Join(tempdir, "plain.gguf")).To(BeAnExistingFile()) + }) + + // Half the point of the change is that the empty-base path costs no fetch, + // and an assertion that nothing was fetched is worthless unless something + // could have been. So the control runs the same install with a url: pointing + // at a base config that is not there: it must fail, proving a url IS read on + // this path, and only then does the same fixture without the url passing + // mean the read was skipped rather than silently succeeding. + Describe("the base config fetch", func() { + var missing string + + BeforeEach(func() { + missing = "file://" + filepath.Join(tempdir, "no-such-base.yaml") + Expect(filepath.Join(tempdir, "no-such-base.yaml")).ToNot(BeAnExistingFile()) + }) + + It("is attempted when the entry declares a url", func() { + e := gallery.GalleryModel{Overrides: map[string]any{"backend": "ds4"}} + e.Name = "with-url" + e.URL = missing + newGallery(e) + + // If this ever starts passing, the control has stopped controlling + // and the spec below proves nothing. + Expect(install("with-url", gallery.GalleryModel{})).To(HaveOccurred()) + }) + + It("is skipped entirely when the entry declares none", func() { + e := gallery.GalleryModel{Overrides: map[string]any{"backend": "ds4"}} + e.Name = "without-url" + newGallery(e) + + Expect(install("without-url", gallery.GalleryModel{})).To(Succeed()) + Expect(installedConfig("without-url")["backend"]).To(Equal("ds4")) + }) + }) + + Describe("an entry with nothing to install", func() { + // No url, no config_file, no overrides and no files. Accepting this + // would write an empty model directory and report success, which is the + // authoring mistake the relaxation would otherwise hide. + emptyEntry := func() gallery.GalleryModel { + e := gallery.GalleryModel{} + e.Name = "hollow" + // urls: is the informational link list. It reads like a payload and + // is not one. + e.URLs = []string{"https://huggingface.co/example/hollow"} + return e + } + + It("is refused rather than installed empty", func() { + newGallery(emptyEntry()) + + err := install("hollow", gallery.GalleryModel{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("installs nothing")) + Expect(err.Error()).To(ContainSubstring("hollow")) + Expect(filepath.Join(tempdir, "hollow.yaml")).ToNot(BeAnExistingFile()) + }) + + It("is accepted when the caller's request supplies the payload", func() { + // The request's overrides are merged into the install exactly as the + // entry's own are, so a caller who brings them really has asked for + // something installable. + newGallery(emptyEntry()) + + req := gallery.GalleryModel{Overrides: map[string]any{"backend": "llama-cpp"}} + Expect(install("hollow", req)).To(Succeed()) + Expect(installedConfig("hollow")["backend"]).To(Equal("llama-cpp")) + }) + }) + + // The two older paths are meant to be untouched by the relaxation. + Describe("the pre-existing paths", func() { + It("still installs an entry described by an inline config_file", func() { + e := gallery.GalleryModel{ConfigFile: map[string]any{"backend": "llama-cpp"}} + e.Name = "inline" + newGallery(e) + + Expect(install("inline", gallery.GalleryModel{})).To(Succeed()) + Expect(installedConfig("inline")["backend"]).To(Equal("llama-cpp")) + }) + + It("still installs an entry described by a url", func() { + payload, err := yaml.Marshal(gallery.ModelConfig{ + Name: "fetched", + ConfigFile: "backend: vllm\n", + }) + Expect(err).ToNot(HaveOccurred()) + payloadPath := filepath.Join(tempdir, "payload.yaml") + Expect(os.WriteFile(payloadPath, payload, 0600)).To(Succeed()) + + e := gallery.GalleryModel{} + e.Name = "from-url" + e.URL = "file://" + payloadPath + newGallery(e) + + Expect(install("from-url", gallery.GalleryModel{})).To(Succeed()) + Expect(installedConfig("from-url")["backend"]).To(Equal("vllm")) + }) + }) + + // One of the entries that shipped uninstallable, driven through the real + // install path rather than only checked as text. Its files: are swapped for + // a local one because the real ones are gigabytes on HuggingFace; its + // overrides: are the catalog's own, so this proves the authored payload + // lands. + It("installs a previously broken index entry once the empty base is allowed", func() { + entries, err := loadGalleryIndex() + Expect(err).ToNot(HaveOccurred()) + + var e gallery.GalleryModel + for _, candidate := range entries { + if candidate.Name == "liquidai_lfm2-1.2b-rag" { + e = candidate + break + } + } + Expect(e.Name).To(Equal("liquidai_lfm2-1.2b-rag")) + // The defect: no base config of any kind, which used to be fatal. + Expect(e.URL).To(BeEmpty()) + Expect(e.ConfigFile).To(BeEmpty()) + Expect(e.Overrides).ToNot(BeEmpty()) + + e.AdditionalFiles = []gallery.File{localWeights("LiquidAI_LFM2-1.2B-RAG-Q4_K_M.gguf")} + newGallery(e) + + Expect(install(e.Name, gallery.GalleryModel{})).To(Succeed()) + cfg := installedConfig(e.Name) + Expect(cfg["name"]).To(Equal(e.Name)) + // The catalog's own overrides, verbatim, laid over the empty base. + Expect(cfg["parameters"]).To(Equal(e.Overrides["parameters"])) + Expect(cfg["known_usecases"]).To(Equal(e.Overrides["known_usecases"])) + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index 2bb748e5c..f3184648b 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -410,7 +410,28 @@ func InstallModelFromGallery( // Prompt Template Skipped for now - I expect in this mode that they will be delivered as files. } } else { - return fmt.Errorf("invalid gallery model %+v", model) + // An entry that names no base config describes itself entirely + // through overrides: and files:, so the base is simply empty. + // + // This is what the several hundred entries pointing at + // gallery/virtual.yaml were already getting. That stub carries only + // a name, a description and a license, all three of which are + // overwritten from the gallery entry a few lines up, and overrides + // reach InstallModel as a separate argument rather than being merged + // into the fetched config. So the fetch bought nothing beyond a + // round trip to GitHub on every install, and an author who omits the + // key is asking for exactly the same thing. + if !model.installsSomething(req) { + return fmt.Errorf("gallery model %q installs nothing: it declares no url, no config_file, no overrides and no files", model.Name) + } + config = ModelConfig{ + Description: model.Description, + License: model.License, + Name: model.Name, + Files: make([]File, 0), // Real values get added below, must be blank + // URLs are appended once below for every branch, as in the + // config_file case above. + } } if record != nil { diff --git a/core/gallery/models_types.go b/core/gallery/models_types.go index 922ac1703..a623ff535 100644 --- a/core/gallery/models_types.go +++ b/core/gallery/models_types.go @@ -27,6 +27,27 @@ type GalleryModel struct { Variants []Variant `json:"variants,omitempty" yaml:"variants,omitempty"` } +// installsSomething reports whether this entry, combined with the caller's +// request, would put anything on disk. +// +// It exists to keep an authoring mistake loud. Once an entry with no url and no +// config_file is accepted as an empty base, the only thing separating a +// deliberate overrides-only entry from a half-written stanza is whether it +// carries a payload at all. Without this check the stub would install cleanly +// and leave a model directory holding a config naming no weights, which fails +// far from the entry that caused it. +// +// The request is counted because its overrides and files are merged into the +// install exactly as the entry's own are, so a caller supplying them really has +// asked for something installable. +// +// An entry with a url or a config_file never reaches this: it has a base config +// to install, however thin. +func (m *GalleryModel) installsSomething(req GalleryModel) bool { + return len(m.Overrides) > 0 || len(m.AdditionalFiles) > 0 || + len(req.Overrides) > 0 || len(req.AdditionalFiles) > 0 +} + func (m *GalleryModel) GetInstalled() bool { return m.Installed } diff --git a/core/gallery/variants_lint_test.go b/core/gallery/variants_lint_test.go index 5e925f0ff..c5e0bda30 100644 --- a/core/gallery/variants_lint_test.go +++ b/core/gallery/variants_lint_test.go @@ -75,45 +75,43 @@ func checkVariantReferences(entries []gallery.GalleryModel) []variantViolation { return violations } -// checkVariantTargetsInstallable verifies every entry referenced as a variant -// carries the payload an install needs, which is a non-empty url or a non-empty -// config_file. +// checkEntriesInstallSomething verifies every entry would actually put +// something on disk. // -// This mirrors the precondition InstallModelFromGallery's applyModel enforces: -// with neither field it has nothing to build a config from and fails with -// "invalid gallery model". Structural validity is not enough, because an entry -// can exist, declare no variants of its own, and still be uninstallable. That -// gap is how a grouping shipped whose every target failed on click: the parent -// resolved correctly and then routed the install into a dead entry. +// It replaces an earlier rule that demanded a url: or a config_file: from every +// variant target. That demand no longer holds: applyModel now treats an entry +// declaring neither as an empty base config, which is precisely what the many +// entries pointing at gallery/virtual.yaml were already getting, minus the +// fetch. Requiring one of the two fields would now reject perfectly good +// authoring. // -// The message names the parent, the target and what is missing, because whoever -// hits this is reading a gallery entry and has no reason to know applyModel -// exists. -func checkVariantTargetsInstallable(entries []gallery.GalleryModel) []variantViolation { - byName := indexEntriesByName(entries) +// What survives is the weaker invariant the relaxation left exposed. An entry +// with no base config, no overrides: and no files: names nothing to download +// and nothing to configure, so installing it yields an empty model directory. +// That is an authoring mistake, and it is worth catching in the catalog rather +// than on someone's machine. +// +// The rule covers every entry, not only variant targets, because the hazard has +// nothing to do with variants: it is a half-written stanza, and a parent entry +// can be one just as easily as a target. +// entryInstallsSomething restates applyModel's acceptance rule in the terms an +// author reads: a base config, or a payload to lay over an empty one. +func entryInstallsSomething(e gallery.GalleryModel) bool { + return len(e.URL) > 0 || len(e.ConfigFile) > 0 || len(e.Overrides) > 0 || len(e.AdditionalFiles) > 0 +} + +func checkEntriesInstallSomething(entries []gallery.GalleryModel) []variantViolation { var violations []variantViolation for _, e := range entries { - if !e.HasVariants() { + if entryInstallsSomething(e) { continue } - for _, v := range e.Variants { - target, ok := byName[v.Model] - if !ok { - // checkVariantReferences already reports the dangling name, and - // reporting it twice buries two distinct rules under duplicates. - continue - } - if len(target.URL) == 0 && len(target.ConfigFile) == 0 { - violations = append(violations, variantViolation{ - Entry: e.Name, - Variant: v.Model, - Detail: fmt.Sprintf("entry %q is not installable on its own: it declares neither url: nor config_file:, "+ - "so installing it fails with \"invalid gallery model\". Give it the url: its family uses "+ - "(commonly github:mudler/LocalAI/gallery/virtual.yaml@master) or an inline config_file:. "+ - "urls: (plural) is informational only and does not satisfy this", v.Model), - }) - } - } + violations = append(violations, variantViolation{ + Entry: e.Name, + Detail: fmt.Sprintf("entry %q installs nothing: it declares no url:, no config_file:, no overrides: and no files:, "+ + "so installing it would leave an empty model directory. Give it the payload it is missing. "+ + "Note that urls: (plural) is the informational link list and is not a payload", e.Name), + }) } return violations } @@ -182,38 +180,58 @@ var _ = Describe("gallery variant lint helpers", func() { ) Expect(checkVariantReferences(entries)).To(BeEmpty()) - Expect(checkVariantTargetsInstallable(entries)).To(BeEmpty()) + Expect(checkEntriesInstallSomething(entries)).To(BeEmpty()) }) - Describe("checkVariantTargetsInstallable", func() { - // The defect this rule exists for: a target carrying everything except - // the one field applyModel reads. - uninstallable := func(name string) gallery.GalleryModel { + Describe("checkEntriesInstallSomething", func() { + // The shape the rule exists for: a stanza that got as far as a name and + // stopped. + empty := func(name string) gallery.GalleryModel { e := gallery.GalleryModel{} e.Name = name - // urls: is the informational HuggingFace link list, and the entry - // that shipped broken had exactly this and nothing else. It must - // not be mistaken for url:. + // urls: is the informational HuggingFace link list. It reads like a + // payload and is not one, so a stub carrying only this must still be + // flagged. e.URLs = []string{"https://huggingface.co/example/" + name} - e.Overrides = map[string]any{"backend": "ds4"} return e } - It("flags a target with neither url nor config_file", func() { + It("flags an entry with no url, no config_file, no overrides and no files", func() { entries := variantFixture( entryWithVariants("base", "u://base", gallery.Variant{Model: "dead"}), - uninstallable("dead"), + empty("dead"), ) - violations := checkVariantTargetsInstallable(entries) + violations := checkEntriesInstallSomething(entries) Expect(violations).To(HaveLen(1)) - Expect(violations[0].Entry).To(Equal("base")) - Expect(violations[0].Variant).To(Equal("dead")) - Expect(violations[0].Detail).To(ContainSubstring("neither url: nor config_file:")) - Expect(violations[0].Detail).To(ContainSubstring("invalid gallery model")) + Expect(violations[0].Entry).To(Equal("dead")) + Expect(violations[0].Detail).To(ContainSubstring("installs nothing")) }) - It("accepts a target described by an inline config_file rather than a url", func() { + It("accepts an entry carrying only overrides, which applyModel installs on an empty base", func() { + target := gallery.GalleryModel{Overrides: map[string]any{"backend": "ds4"}} + target.Name = "overrides-only" + entries := variantFixture( + entryWithVariants("base", "u://base", gallery.Variant{Model: "overrides-only"}), + target, + ) + + Expect(checkEntriesInstallSomething(entries)).To(BeEmpty()) + }) + + It("accepts an entry carrying only files", func() { + target := gallery.GalleryModel{} + target.Name = "files-only" + target.AdditionalFiles = []gallery.File{{Filename: "weights.gguf", URI: "u://weights"}} + entries := variantFixture( + entryWithVariants("base", "u://base", gallery.Variant{Model: "files-only"}), + target, + ) + + Expect(checkEntriesInstallSomething(entries)).To(BeEmpty()) + }) + + It("accepts an entry described by an inline config_file rather than a url", func() { target := gallery.GalleryModel{ConfigFile: map[string]any{"backend": "llama-cpp"}} target.Name = "inline" entries := variantFixture( @@ -221,16 +239,7 @@ var _ = Describe("gallery variant lint helpers", func() { target, ) - Expect(checkVariantTargetsInstallable(entries)).To(BeEmpty()) - }) - - It("leaves an unknown target to checkVariantReferences rather than reporting it twice", func() { - entries := variantFixture( - entryWithVariants("base", "u://base", gallery.Variant{Model: "ghost"}), - ) - - Expect(checkVariantTargetsInstallable(entries)).To(BeEmpty()) - Expect(checkVariantReferences(entries)).To(HaveLen(1)) + Expect(checkEntriesInstallSomething(entries)).To(BeEmpty()) }) It("reports every breach in one pass rather than stopping at the first", func() { @@ -239,11 +248,11 @@ var _ = Describe("gallery variant lint helpers", func() { gallery.Variant{Model: "dead-a"}, gallery.Variant{Model: "dead-b"}, ), - uninstallable("dead-a"), - uninstallable("dead-b"), + empty("dead-a"), + empty("dead-b"), ) - Expect(checkVariantTargetsInstallable(entries)).To(HaveLen(2)) + Expect(checkEntriesInstallSomething(entries)).To(HaveLen(2)) }) }) @@ -304,8 +313,13 @@ var _ = Describe("gallery/index.yaml variant invariants", Ordered, func() { Expect(v).To(BeEmpty(), formatViolations(v)) }) - It("references only entries that are installable on their own", func() { - v := checkVariantTargetsInstallable(entries) + // Gallery-wide, not variants-only. The rule this replaced was scoped to + // variant targets because the index carried nine unrelated entries it would + // have failed on. Those nine declared overrides and files but no url, and + // they install cleanly on an empty base now, so there is nothing left to + // exempt and the gate covers the whole catalog in one step. + It("contains no entry that would install nothing", func() { + v := checkEntriesInstallSomething(entries) Expect(v).To(BeEmpty(), formatViolations(v)) }) }) @@ -357,15 +371,19 @@ var _ = Describe("gallery/index.yaml deepseek-v4-flash resolution", Ordered, fun resolved, selected, err := gallery.ResolveVariant(models, entry, env, "") Expect(err).ToNot(HaveOccurred()) - Expect(len(resolved.URL) > 0 || len(resolved.ConfigFile) > 0).To(BeTrue(), - "resolved entry %q (variant %q) has neither url nor config_file, so InstallModelFromGallery would fail with \"invalid gallery model\"", + Expect(entryInstallsSomething(*resolved)).To(BeTrue(), + "resolved entry %q (variant %q) carries no payload, so InstallModelFromGallery would refuse it", resolved.Name, selected.Model) }) // Pinning reaches each target directly, which is what actually proves all // four are installable: every one of them is resolved, not just whichever - // the ranking happens to prefer. This is the spec that fails on the - // unfixed index. + // the ranking happens to prefer. + // + // None of the four declares a url: any more. They describe themselves + // entirely through overrides: and files:, which applyModel lays over an + // empty base, so this also pins that dropping the urls left them + // installable. It("yields an installable entry for every declared variant pin", func() { env := gallery.ResolveEnv{ AvailableMemory: 512 << 30, @@ -375,8 +393,8 @@ var _ = Describe("gallery/index.yaml deepseek-v4-flash resolution", Ordered, fun for _, v := range entry.Variants { resolved, _, err := gallery.ResolveVariant(models, entry, env, v.Model) Expect(err).ToNot(HaveOccurred(), "pinning %q", v.Model) - Expect(len(resolved.URL) > 0 || len(resolved.ConfigFile) > 0).To(BeTrue(), - "variant %q has neither url nor config_file", v.Model) + Expect(resolved.URL).To(BeEmpty(), "variant %q should reach the empty-base path, not a fetch", v.Model) + Expect(entryInstallsSomething(*resolved)).To(BeTrue(), "variant %q carries no payload", v.Model) } }) }) diff --git a/gallery/index.yaml b/gallery/index.yaml index a6022de92..70205d322 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -36884,7 +36884,6 @@ sha256: c61cbb396e2a8175d8b2da51f0fdac885a4ccd22c9f64dafa5aa2c455dc8a507 uri: huggingface://unsloth/LTX-2.3-GGUF/text_encoders/ltx-2.3-22b-distilled_embeddings_connectors.safetensors - name: deepseek-v4-flash-q2 - url: "github:mudler/LocalAI/gallery/virtual.yaml@master" description: | DeepSeek V4 Flash (IQ2XXS GGUF, ~81 GB) - only loadable via the ds4 backend. Requires >=128 GB RAM. Metal (Darwin) or CUDA (Linux). @@ -36906,7 +36905,6 @@ sha256: 31598c67c8b8744d3bcebcd19aa62253c6dc43cef3b8adf9f593656c9e86fd8c uri: huggingface://antirez/deepseek-v4-gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2.gguf - name: deepseek-v4-flash-q2-q4 - url: "github:mudler/LocalAI/gallery/virtual.yaml@master" description: | DeepSeek V4 Flash (mixed q2/q4 GGUF, ~91 GB) - only loadable via the ds4 backend. The last 6 expert layers are kept at Q4_K (the rest IQ2XXS), trading a little @@ -36930,7 +36928,6 @@ sha256: edabc92af63ad8b139f00087fbfc10a4072f37b7597f4fd9ad1dfa6f83002396 uri: huggingface://antirez/deepseek-v4-gguf/DeepSeek-V4-Flash-Layers37-42Q4KExperts-OtherExpertLayersIQ2XXSGateUp-Q2KDown-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-fixed.gguf - name: deepseek-v4-flash-q4-ssd - url: "github:mudler/LocalAI/gallery/virtual.yaml@master" description: | DeepSeek V4 Flash (full 4-bit experts GGUF, ~153 GB) - only loadable via the ds4 backend, with SSD streaming enabled so it runs on a 128 GB machine even @@ -36958,7 +36955,6 @@ sha256: 39e5de72ac544fdd5ffaf83ec28e36aaf3341b145235488e67d59400bbb3af55 uri: huggingface://antirez/deepseek-v4-gguf/DeepSeek-V4-Flash-Q4KExperts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2.gguf - name: deepseek-v4-flash-q2-mtp - url: "github:mudler/LocalAI/gallery/virtual.yaml@master" description: | DeepSeek V4 Flash (IQ2XXS GGUF, ~81 GB) paired with the optional MTP speculative-decoding weights (~3.5 GB) for a slight speedup. Only loadable