From 561d623976218f811ed4fbb8b3217a6137102e57 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 7 Sep 2026 20:55:42 +0200 Subject: [PATCH] feat(gallery): add EXL3 vllm.cpp models (#11912) * docs: design EXL3 gallery entries Define the pinned Qwen3.8 and DeepSeek V4 Flash entries before changing the gallery. Assisted-by: Codex:gpt-5 [web] * feat(gallery): add EXL3 vllm.cpp models Add pinned Qwen3.8 and DeepSeek V4 Flash configurations. Bind the staged DFlash companion path so vllm.cpp opens LocalAI's managed snapshot. Assisted-by: Codex:gpt-5 --------- Co-authored-by: Ettore Di Giacinto --- backend/go/vllm-cpp/options.go | 31 +++ backend/go/vllm-cpp/options_test.go | 38 ++++ core/gallery/exl3_gallery_test.go | 101 +++++++++ .../specs/2026-09-07-exl3-gallery-design.md | 99 +++++++++ gallery/index.yaml | 196 ++++++++++++++++++ 5 files changed, 465 insertions(+) create mode 100644 backend/go/vllm-cpp/options_test.go create mode 100644 core/gallery/exl3_gallery_test.go create mode 100644 docs/superpowers/specs/2026-09-07-exl3-gallery-design.md diff --git a/backend/go/vllm-cpp/options.go b/backend/go/vllm-cpp/options.go index 86717e7ac..b75e5a7fc 100644 --- a/backend/go/vllm-cpp/options.go +++ b/backend/go/vllm-cpp/options.go @@ -128,9 +128,40 @@ func parseOptions(opts *pb.ModelOptions) loadOptions { lo := loadOptions{} applyOptionsList(&lo, opts.GetOptions()) applyEngineArgs(&lo, opts.GetEngineArgs()) + applyDraftModelOption(&lo, opts.GetOptions()) return lo } +// applyDraftModelOption binds a managed companion snapshot after engine_args +// has supplied the speculative document. Companion paths do not exist until +// LocalAI materializes the artifact, so they must replace the gallery's static +// repository reference without disturbing the method or token budget. +func applyDraftModelOption(lo *loadOptions, options []string) { + if strings.TrimSpace(lo.speculativeConfig) == "" { + return + } + var draftModel string + for _, option := range options { + key, value, found := strings.Cut(option, ":") + if found && strings.TrimSpace(key) == "draft_model" { + draftModel = strings.TrimSpace(value) + } + } + if draftModel == "" { + return + } + + var spec map[string]any + if err := json.Unmarshal([]byte(lo.speculativeConfig), &spec); err != nil { + return + } + spec["model"] = draftModel + encoded, err := json.Marshal(spec) + if err == nil { + lo.speculativeConfig = string(encoded) + } +} + // applyOptionsList reads the legacy free-form "key:value" list. strings.Cut // splits on the FIRST colon only, so a JSON object value survives intact. func applyOptionsList(lo *loadOptions, options []string) { diff --git a/backend/go/vllm-cpp/options_test.go b/backend/go/vllm-cpp/options_test.go new file mode 100644 index 000000000..11b222b4a --- /dev/null +++ b/backend/go/vllm-cpp/options_test.go @@ -0,0 +1,38 @@ +package main + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" +) + +var _ = Describe("managed DFlash companion options", func() { + It("replaces only the draft model in an existing speculative configuration", func() { + managedPath := ".artifacts/huggingface/0123456789abcdef/snapshot" + lo := parseOptions(&pb.ModelOptions{ + Options: []string{"draft_model:" + managedPath}, + EngineArgs: `{ + "speculative_config": { + "method": "dflash", + "model": "Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw", + "num_speculative_tokens": 7 + } + }`, + }) + + Expect(lo.speculativeConfig).To(MatchJSON(`{ + "method": "dflash", + "model": ".artifacts/huggingface/0123456789abcdef/snapshot", + "num_speculative_tokens": 7 + }`)) + }) + + It("ignores a draft companion when speculative decoding is not configured", func() { + lo := parseOptions(&pb.ModelOptions{ + Options: []string{"draft_model:.artifacts/huggingface/0123456789abcdef/snapshot"}, + }) + + Expect(lo.speculativeConfig).To(BeEmpty()) + }) +}) diff --git a/core/gallery/exl3_gallery_test.go b/core/gallery/exl3_gallery_test.go new file mode 100644 index 000000000..5c55be57e --- /dev/null +++ b/core/gallery/exl3_gallery_test.go @@ -0,0 +1,101 @@ +package gallery_test + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "gopkg.in/yaml.v3" +) + +func remarshal(value any, target any) error { + data, err := yaml.Marshal(value) + if err != nil { + return err + } + return yaml.Unmarshal(data, target) +} + +var _ = Describe("EXL3 gallery entries", func() { + It("pins the four full repositories and configures the Qwen DFlash companion", func() { + entries, err := gallery.ReadConfigFile[[]gallery.GalleryModel](filepath.Join("..", "..", "gallery", "index.yaml")) + Expect(err).ToNot(HaveOccurred()) + + byName := make(map[string]gallery.GalleryModel, len(*entries)) + for _, entry := range *entries { + byName[entry.Name] = entry + } + + expected := map[string]struct { + repo string + revision string + }{ + "qwen3.8-27b-exl3-vllm-cpp": { + repo: "Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw", revision: "19441ac874c4018295da848e250f23511361cda4", + }, + "qwen3.8-27b-dflash2-exl3-vllm-cpp": { + repo: "Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw", revision: "19441ac874c4018295da848e250f23511361cda4", + }, + "deepseek-v4-flash-spark-exl3-vllm-cpp": { + repo: "0xSero/deepseek-v4-flash-0731-spark", revision: "ce5ff0f1efb2e184aafc759d281bfae47d3a359c", + }, + "deepseek-v4-flash-exl3-3bpw-vllm-cpp": { + repo: "0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw", revision: "e0bf84ac76a5100e8790c22ad10b70b1e2d06d71", + }, + } + + for name, want := range expected { + entry, found := byName[name] + Expect(found).To(BeTrue(), "missing gallery entry %q", name) + Expect(entry.Tags).To(ContainElements("vllm-cpp", "exl3", "gpu", "cuda"), name) + Expect(entry.Overrides).To(HaveKeyWithValue("backend", "vllm-cpp"), name) + cfg := config.ModelConfig{} + Expect(remarshal(entry.Overrides, &cfg)).To(Succeed(), name) + Expect(cfg.Artifacts).ToNot(BeEmpty(), name) + Expect(cfg.Artifacts[0].Source.Repo).To(Equal(want.repo), name) + Expect(cfg.Artifacts[0].Source.Revision).To(Equal(want.revision), name) + } + + plain := byName["qwen3.8-27b-exl3-vllm-cpp"] + Expect(plain.Tags).ToNot(ContainElement("dflash")) + dflashTags := 0 + for name := range expected { + if contains(byName[name].Tags, "dflash") { + dflashTags++ + } + } + Expect(dflashTags).To(Equal(1)) + + dflash := byName["qwen3.8-27b-dflash2-exl3-vllm-cpp"] + Expect(dflash.Tags).To(ContainElement("dflash")) + Expect(dflash.Variants).To(ConsistOf(gallery.Variant{Model: "qwen3.8-27b-exl3-vllm-cpp"})) + cfg := config.ModelConfig{} + Expect(remarshal(dflash.Overrides, &cfg)).To(Succeed()) + Expect(cfg.ContextSize).To(HaveValue(Equal(8192))) + Expect(cfg.EngineArgs).To(HaveKeyWithValue("num_blocks", 2048)) + Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_seqs", 8)) + Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_batched_tokens", 16384)) + Expect(cfg.EngineArgs).To(HaveKeyWithValue("enable_prefix_caching", false)) + spec, ok := cfg.EngineArgs["speculative_config"].(map[string]any) + Expect(ok).To(BeTrue()) + Expect(spec).To(HaveKeyWithValue("method", "dflash")) + Expect(spec).To(HaveKeyWithValue("num_speculative_tokens", 7)) + Expect(cfg.Artifacts).To(HaveLen(2)) + Expect(cfg.Artifacts[1].Name).To(Equal("draft_model")) + Expect(cfg.Artifacts[1].Target).To(Equal("companion")) + Expect(cfg.Artifacts[1].Source.Repo).To(Equal("Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw")) + Expect(cfg.Artifacts[1].Source.Revision).To(Equal("4f0436269bca761b071f05319e8e04a87cc633f9")) + }) +}) + +func contains(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} diff --git a/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md b/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md new file mode 100644 index 000000000..b4175655f --- /dev/null +++ b/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md @@ -0,0 +1,99 @@ +# EXL3 gallery entries + +## Goal + +Add four gallery entries that expose the EXL3 configurations validated or +tracked by `vllm.cpp`. Pin each Hugging Face artifact to the revision recorded +by its source or benchmark evidence. + +## Entries + +### Qwen3.8 target + +Add `qwen3.8-27b-exl3-vllm-cpp` for +`Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw`. This entry serves the target without a +draft model. + +Use revision `19441ac874c4018295da848e250f23511361cda4`. Configure an 8,192-token +context, 2,048 cache blocks, eight sequences, and 16,384 batched tokens. Disable +prefix caching to match the measured serving configuration. + +### Qwen3.8 with DFlash2 + +Add `qwen3.8-27b-dflash2-exl3-vllm-cpp`. This entry stages the Qwen3.8 target +and `Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw` at revision +`4f0436269bca761b071f05319e8e04a87cc633f9`. + +Configure the `dflash` method with seven speculative tokens. Use the shipped +paged draft route. Apply the same serving limits as the target-only entry. + +Tag this entry with `dflash` because it enables speculative decoding. Declare +the target-only entry as its variant. LocalAI can then prefer the faster entry +when the host supports it. + +### DeepSeek V4 Flash for Spark + +Add `deepseek-v4-flash-spark-exl3-vllm-cpp` for +`0xSero/deepseek-v4-flash-0731-spark`. Use the current repository revision, +`ce5ff0f1efb2e184aafc759d281bfae47d3a359c`. State that the `vllm.cpp` +runtime record used the older revision `22f28d32b9b29b4352eaa380ff8c2c170b2847ab`. + +Describe the entry as a Spark and GB10-oriented REAP-K216 checkpoint. State its +large memory requirement and CUDA requirement. Do not claim a completed speed +or correctness gate that the source record does not contain. + +### DeepSeek V4 Flash 3.0 bpw + +Add `deepseek-v4-flash-exl3-3bpw-vllm-cpp` for +`0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw`. Use the current repository +revision, `e0bf84ac76a5100e8790c22ad10b70b1e2d06d71`. + +Tag and describe this entry as experimental. The model card states that the +artifact is structurally complete, but end-to-end generation has not passed. +Keep this entry separate from the Spark entry because the repositories use +different layouts and have different runtime evidence. + +## Artifact staging + +Use LocalAI's Hugging Face artifact source for each repository. Stage complete +model repositories because these safetensors checkpoints need configuration, +tokenizer, index, and weight files. + +Assign the Qwen draft artifact to a companion target. Pass its staged path in +the `vllm-cpp` speculative configuration. Do not download files through backend +startup logic. + +## User-visible metadata + +Use the `vllm-cpp`, `exl3`, `gpu`, and `cuda` tags on all four entries. Add +architecture, reasoning, tool-calling, and speculative-decoding tags only when +the configured model supports them. + +Descriptions must distinguish measured results from unresolved work. The Qwen +DFlash2 description can cite the measured configuration and throughput. The +DeepSeek descriptions must not imply an end-to-end validation that does not +exist. + +## Validation + +Run the gallery schema and focused gallery tests. Add a focused test if the +artifact or variant structure is not already covered. + +Validate these properties: + +- Every name is unique. +- Every variant points to an existing entry. +- Each Hugging Face source has a pinned revision. +- The DFlash2 entry stages both repositories and passes the draft path. +- Only the configured DFlash2 entry has the `dflash` tag. +- YAML parsing and gallery loading succeed. + +No model download or GPU benchmark is part of this LocalAI change. The +`vllm.cpp` evidence supplies the runtime record. + +## Out of scope + +- Changes to the `vllm-cpp` backend binaries. +- New EXL3 kernels or model loaders. +- New benchmark claims. +- Gallery entries for unselected EXL3 bit widths. diff --git a/gallery/index.yaml b/gallery/index.yaml index 04e1c84e9..db4773975 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -14484,6 +14484,202 @@ - vae/** parameters: model: meituan-longcat/LongCat-Video-Avatar-1.5 +- name: qwen3.8-27b-exl3-vllm-cpp + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw + - https://github.com/mudler/vllm.cpp + description: | + Qwen3.8-27B EXL3 3.5bpw served by vllm.cpp, LocalAI's C++ vLLM-style + runtime. The published checkpoint generates on CUDA and measured 16.7 + tokens/s on GB10 with the pinned revision and the limits configured here. + + This is the target-only setup. Use the DFlash2 variant for the measured + speculative-decoding configuration. The entry downloads the complete + revision-pinned repository, including its configuration, tokenizer, index, + and safetensors shards. + tags: + - llm + - qwen + - qwen3.8 + - exl3 + - vllm-cpp + - reasoning + - tool-calling + - gpu + - cuda + size: 15.4GB + last_checked: "2026-09-07" + overrides: + backend: vllm-cpp + known_usecases: + - chat + - completion + function: + grammar: + disable: true + template: + use_tokenizer_template: true + context_size: 8192 + engine_args: + num_blocks: 2048 + max_num_seqs: 8 + max_num_batched_tokens: 16384 + enable_prefix_caching: false + parameters: + model: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw + artifacts: + - name: model + target: model + source: + type: huggingface + repo: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw + revision: 19441ac874c4018295da848e250f23511361cda4 +- name: qwen3.8-27b-dflash2-exl3-vllm-cpp + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw + - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw + - https://github.com/mudler/vllm.cpp + description: | + Qwen3.8-27B EXL3 with its EXL3 DFlash2 companion, served by vllm.cpp. On + GB10 this pinned pair measured 48.7 tokens/s at a seven-token draft budget, + versus 16.7 tokens/s target-only, with token-identical greedy output. + + LocalAI stages both complete Hugging Face repositories before load. The + content-addressed companion snapshot is passed to the backend as the draft + model, while the speculative method and seven-token budget remain fixed. + tags: + - llm + - qwen + - qwen3.8 + - exl3 + - vllm-cpp + - speculative-decoding + - dflash + - reasoning + - tool-calling + - gpu + - cuda + size: 16.9GB + last_checked: "2026-09-07" + variants: + - model: qwen3.8-27b-exl3-vllm-cpp + overrides: + backend: vllm-cpp + known_usecases: + - chat + - completion + function: + grammar: + disable: true + template: + use_tokenizer_template: true + context_size: 8192 + engine_args: + num_blocks: 2048 + max_num_seqs: 8 + max_num_batched_tokens: 16384 + enable_prefix_caching: false + speculative_config: + method: dflash + model: Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw + num_speculative_tokens: 7 + parameters: + model: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw + artifacts: + - name: model + target: model + source: + type: huggingface + repo: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw + revision: 19441ac874c4018295da848e250f23511361cda4 + - name: draft_model + target: companion + source: + type: huggingface + repo: Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw + revision: 4f0436269bca761b071f05319e8e04a87cc633f9 +- name: deepseek-v4-flash-spark-exl3-vllm-cpp + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/0xSero/deepseek-v4-flash-0731-spark + - https://github.com/mudler/vllm.cpp + description: | + DeepSeek V4 Flash's Spark and GB10-oriented REAP-K216 EXL3 checkpoint, + served by vllm.cpp. It needs CUDA and roughly 100 GiB for its large + rank-sliced checkpoint. + + The repository is pinned to its latest recorded revision. vllm.cpp's + existing runtime evidence measured the older 22f28d32b9b29b4352eaa380ff8c2c170b2847ab + revision; this entry does not claim that the newer revision has passed the + same end-to-end gate. + tags: + - llm + - deepseek + - deepseek-v4 + - exl3 + - vllm-cpp + - gpu + - cuda + - gb10 + size: 100GB + last_checked: "2026-09-07" + overrides: + backend: vllm-cpp + known_usecases: + - chat + - completion + template: + use_tokenizer_template: true + parameters: + model: 0xSero/deepseek-v4-flash-0731-spark + artifacts: + - name: model + target: model + source: + type: huggingface + repo: 0xSero/deepseek-v4-flash-0731-spark + revision: ce5ff0f1efb2e184aafc759d281bfae47d3a359c +- name: deepseek-v4-flash-exl3-3bpw-vllm-cpp + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw + - https://github.com/mudler/vllm.cpp + description: | + Experimental non-Spark DeepSeek V4 Flash EXL3 3.0bpw checkpoint served by + vllm.cpp. This is the complete, non-REAP layout and requires a large + multi-GPU CUDA system. + + The publisher describes the artifact as structurally complete but has not + passed end-to-end generation. Treat this entry as an integration target, + not as a correctness- or performance-gated configuration. + tags: + - llm + - deepseek + - deepseek-v4 + - exl3 + - vllm-cpp + - experimental + - gpu + - cuda + last_checked: "2026-09-07" + overrides: + backend: vllm-cpp + known_usecases: + - chat + - completion + template: + use_tokenizer_template: true + parameters: + model: 0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw + artifacts: + - name: model + target: model + source: + type: huggingface + repo: 0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw + revision: e0bf84ac76a5100e8790c22ad10b70b1e2d06d71 - name: qwen3.6-27b-nvfp4-vllm-cpp url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: