feat(gallery): add EXL3 vllm.cpp models (#11912)

* docs: design EXL3 gallery entries Define the pinned Qwen3.8 and DeepSeek V4 Flash entries before changing the gallery. Assisted-by: Codex:gpt-5 [web] * feat(gallery): add EXL3 vllm.cpp models Add pinned Qwen3.8 and DeepSeek V4 Flash configurations. Bind the staged DFlash companion path so vllm.cpp opens LocalAI's managed snapshot. Assisted-by: Codex:gpt-5 --------- Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
2026-10-28 09:08:54 -04:00 · 2026-09-07 20:55:42 +02:00
1 parent aff9db9758
commit 561d623976
@@ -128,9 +128,40 @@ func parseOptions(opts *pb.ModelOptions) loadOptions {
 	lo := loadOptions{}
 	applyOptionsList(&lo, opts.GetOptions())
 	applyEngineArgs(&lo, opts.GetEngineArgs())
+	applyDraftModelOption(&lo, opts.GetOptions())
 	return lo
 }

+// applyDraftModelOption binds a managed companion snapshot after engine_args
+// has supplied the speculative document. Companion paths do not exist until
+// LocalAI materializes the artifact, so they must replace the gallery's static
+// repository reference without disturbing the method or token budget.
+func applyDraftModelOption(lo *loadOptions, options []string) {
+	if strings.TrimSpace(lo.speculativeConfig) == "" {
+		return
+	}
+	var draftModel string
+	for _, option := range options {
+		key, value, found := strings.Cut(option, ":")
+		if found && strings.TrimSpace(key) == "draft_model" {
+			draftModel = strings.TrimSpace(value)
+		}
+	}
+	if draftModel == "" {
+		return
+	}
+
+	var spec map[string]any
+	if err := json.Unmarshal([]byte(lo.speculativeConfig), &spec); err != nil {
+		return
+	}
+	spec["model"] = draftModel
+	encoded, err := json.Marshal(spec)
+	if err == nil {
+		lo.speculativeConfig = string(encoded)
+	}
+}
+
 // applyOptionsList reads the legacy free-form "key:value" list. strings.Cut
 // splits on the FIRST colon only, so a JSON object value survives intact.
 func applyOptionsList(lo *loadOptions, options []string) {
@@ -0,0 +1,38 @@
+package main
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+)
+
+var _ = Describe("managed DFlash companion options", func() {
+	It("replaces only the draft model in an existing speculative configuration", func() {
+		managedPath := ".artifacts/huggingface/0123456789abcdef/snapshot"
+		lo := parseOptions(&pb.ModelOptions{
+			Options: []string{"draft_model:" + managedPath},
+			EngineArgs: `{
+				"speculative_config": {
+					"method": "dflash",
+					"model": "Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw",
+					"num_speculative_tokens": 7
+				}
+			}`,
+		})
+
+		Expect(lo.speculativeConfig).To(MatchJSON(`{
+			"method": "dflash",
+			"model": ".artifacts/huggingface/0123456789abcdef/snapshot",
+			"num_speculative_tokens": 7
+		}`))
+	})
+
+	It("ignores a draft companion when speculative decoding is not configured", func() {
+		lo := parseOptions(&pb.ModelOptions{
+			Options: []string{"draft_model:.artifacts/huggingface/0123456789abcdef/snapshot"},
+		})
+
+		Expect(lo.speculativeConfig).To(BeEmpty())
+	})
+})
@@ -0,0 +1,101 @@
+package gallery_test
+
+import (
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"gopkg.in/yaml.v3"
+)
+
+func remarshal(value any, target any) error {
+	data, err := yaml.Marshal(value)
+	if err != nil {
+		return err
+	}
+	return yaml.Unmarshal(data, target)
+}
+
+var _ = Describe("EXL3 gallery entries", func() {
+	It("pins the four full repositories and configures the Qwen DFlash companion", func() {
+		entries, err := gallery.ReadConfigFile[[]gallery.GalleryModel](filepath.Join("..", "..", "gallery", "index.yaml"))
+		Expect(err).ToNot(HaveOccurred())
+
+		byName := make(map[string]gallery.GalleryModel, len(*entries))
+		for _, entry := range *entries {
+			byName[entry.Name] = entry
+		}
+
+		expected := map[string]struct {
+			repo     string
+			revision string
+		}{
+			"qwen3.8-27b-exl3-vllm-cpp": {
+				repo: "Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw", revision: "19441ac874c4018295da848e250f23511361cda4",
+			},
+			"qwen3.8-27b-dflash2-exl3-vllm-cpp": {
+				repo: "Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw", revision: "19441ac874c4018295da848e250f23511361cda4",
+			},
+			"deepseek-v4-flash-spark-exl3-vllm-cpp": {
+				repo: "0xSero/deepseek-v4-flash-0731-spark", revision: "ce5ff0f1efb2e184aafc759d281bfae47d3a359c",
+			},
+			"deepseek-v4-flash-exl3-3bpw-vllm-cpp": {
+				repo: "0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw", revision: "e0bf84ac76a5100e8790c22ad10b70b1e2d06d71",
+			},
+		}
+
+		for name, want := range expected {
+			entry, found := byName[name]
+			Expect(found).To(BeTrue(), "missing gallery entry %q", name)
+			Expect(entry.Tags).To(ContainElements("vllm-cpp", "exl3", "gpu", "cuda"), name)
+			Expect(entry.Overrides).To(HaveKeyWithValue("backend", "vllm-cpp"), name)
+			cfg := config.ModelConfig{}
+			Expect(remarshal(entry.Overrides, &cfg)).To(Succeed(), name)
+			Expect(cfg.Artifacts).ToNot(BeEmpty(), name)
+			Expect(cfg.Artifacts[0].Source.Repo).To(Equal(want.repo), name)
+			Expect(cfg.Artifacts[0].Source.Revision).To(Equal(want.revision), name)
+		}
+
+		plain := byName["qwen3.8-27b-exl3-vllm-cpp"]
+		Expect(plain.Tags).ToNot(ContainElement("dflash"))
+		dflashTags := 0
+		for name := range expected {
+			if contains(byName[name].Tags, "dflash") {
+				dflashTags++
+			}
+		}
+		Expect(dflashTags).To(Equal(1))
+
+		dflash := byName["qwen3.8-27b-dflash2-exl3-vllm-cpp"]
+		Expect(dflash.Tags).To(ContainElement("dflash"))
+		Expect(dflash.Variants).To(ConsistOf(gallery.Variant{Model: "qwen3.8-27b-exl3-vllm-cpp"}))
+		cfg := config.ModelConfig{}
+		Expect(remarshal(dflash.Overrides, &cfg)).To(Succeed())
+		Expect(cfg.ContextSize).To(HaveValue(Equal(8192)))
+		Expect(cfg.EngineArgs).To(HaveKeyWithValue("num_blocks", 2048))
+		Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_seqs", 8))
+		Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_batched_tokens", 16384))
+		Expect(cfg.EngineArgs).To(HaveKeyWithValue("enable_prefix_caching", false))
+		spec, ok := cfg.EngineArgs["speculative_config"].(map[string]any)
+		Expect(ok).To(BeTrue())
+		Expect(spec).To(HaveKeyWithValue("method", "dflash"))
+		Expect(spec).To(HaveKeyWithValue("num_speculative_tokens", 7))
+		Expect(cfg.Artifacts).To(HaveLen(2))
+		Expect(cfg.Artifacts[1].Name).To(Equal("draft_model"))
+		Expect(cfg.Artifacts[1].Target).To(Equal("companion"))
+		Expect(cfg.Artifacts[1].Source.Repo).To(Equal("Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw"))
+		Expect(cfg.Artifacts[1].Source.Revision).To(Equal("4f0436269bca761b071f05319e8e04a87cc633f9"))
+	})
+})
+
+func contains(values []string, target string) bool {
+	for _, value := range values {
+		if value == target {
+			return true
+		}
+	}
+	return false
+}
@@ -0,0 +1,99 @@
+# EXL3 gallery entries
+
+## Goal
+
+Add four gallery entries that expose the EXL3 configurations validated or
+tracked by `vllm.cpp`. Pin each Hugging Face artifact to the revision recorded
+by its source or benchmark evidence.
+
+## Entries
+
+### Qwen3.8 target
+
+Add `qwen3.8-27b-exl3-vllm-cpp` for
+`Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw`. This entry serves the target without a
+draft model.
+
+Use revision `19441ac874c4018295da848e250f23511361cda4`. Configure an 8,192-token
+context, 2,048 cache blocks, eight sequences, and 16,384 batched tokens. Disable
+prefix caching to match the measured serving configuration.
+
+### Qwen3.8 with DFlash2
+
+Add `qwen3.8-27b-dflash2-exl3-vllm-cpp`. This entry stages the Qwen3.8 target
+and `Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw` at revision
+`4f0436269bca761b071f05319e8e04a87cc633f9`.
+
+Configure the `dflash` method with seven speculative tokens. Use the shipped
+paged draft route. Apply the same serving limits as the target-only entry.
+
+Tag this entry with `dflash` because it enables speculative decoding. Declare
+the target-only entry as its variant. LocalAI can then prefer the faster entry
+when the host supports it.
+
+### DeepSeek V4 Flash for Spark
+
+Add `deepseek-v4-flash-spark-exl3-vllm-cpp` for
+`0xSero/deepseek-v4-flash-0731-spark`. Use the current repository revision,
+`ce5ff0f1efb2e184aafc759d281bfae47d3a359c`. State that the `vllm.cpp`
+runtime record used the older revision `22f28d32b9b29b4352eaa380ff8c2c170b2847ab`.
+
+Describe the entry as a Spark and GB10-oriented REAP-K216 checkpoint. State its
+large memory requirement and CUDA requirement. Do not claim a completed speed
+or correctness gate that the source record does not contain.
+
+### DeepSeek V4 Flash 3.0 bpw
+
+Add `deepseek-v4-flash-exl3-3bpw-vllm-cpp` for
+`0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw`. Use the current repository
+revision, `e0bf84ac76a5100e8790c22ad10b70b1e2d06d71`.
+
+Tag and describe this entry as experimental. The model card states that the
+artifact is structurally complete, but end-to-end generation has not passed.
+Keep this entry separate from the Spark entry because the repositories use
+different layouts and have different runtime evidence.
+
+## Artifact staging
+
+Use LocalAI's Hugging Face artifact source for each repository. Stage complete
+model repositories because these safetensors checkpoints need configuration,
+tokenizer, index, and weight files.
+
+Assign the Qwen draft artifact to a companion target. Pass its staged path in
+the `vllm-cpp` speculative configuration. Do not download files through backend
+startup logic.
+
+## User-visible metadata
+
+Use the `vllm-cpp`, `exl3`, `gpu`, and `cuda` tags on all four entries. Add
+architecture, reasoning, tool-calling, and speculative-decoding tags only when
+the configured model supports them.
+
+Descriptions must distinguish measured results from unresolved work. The Qwen
+DFlash2 description can cite the measured configuration and throughput. The
+DeepSeek descriptions must not imply an end-to-end validation that does not
+exist.
+
+## Validation
+
+Run the gallery schema and focused gallery tests. Add a focused test if the
+artifact or variant structure is not already covered.
+
+Validate these properties:
+
+- Every name is unique.
+- Every variant points to an existing entry.
+- Each Hugging Face source has a pinned revision.
+- The DFlash2 entry stages both repositories and passes the draft path.
+- Only the configured DFlash2 entry has the `dflash` tag.
+- YAML parsing and gallery loading succeed.
+
+No model download or GPU benchmark is part of this LocalAI change. The
+`vllm.cpp` evidence supplies the runtime record.
+
+## Out of scope
+
+- Changes to the `vllm-cpp` backend binaries.
+- New EXL3 kernels or model loaders.
+- New benchmark claims.
+- Gallery entries for unselected EXL3 bit widths.
@@ -14484,6 +14484,202 @@
            - vae/**
    parameters:
      model: meituan-longcat/LongCat-Video-Avatar-1.5
+- name: qwen3.8-27b-exl3-vllm-cpp
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+    - https://github.com/mudler/vllm.cpp
+  description: |
+    Qwen3.8-27B EXL3 3.5bpw served by vllm.cpp, LocalAI's C++ vLLM-style
+    runtime. The published checkpoint generates on CUDA and measured 16.7
+    tokens/s on GB10 with the pinned revision and the limits configured here.
+
+    This is the target-only setup. Use the DFlash2 variant for the measured
+    speculative-decoding configuration. The entry downloads the complete
+    revision-pinned repository, including its configuration, tokenizer, index,
+    and safetensors shards.
+  tags:
+    - llm
+    - qwen
+    - qwen3.8
+    - exl3
+    - vllm-cpp
+    - reasoning
+    - tool-calling
+    - gpu
+    - cuda
+  size: 15.4GB
+  last_checked: "2026-09-07"
+  overrides:
+    backend: vllm-cpp
+    known_usecases:
+      - chat
+      - completion
+    function:
+      grammar:
+        disable: true
+    template:
+      use_tokenizer_template: true
+    context_size: 8192
+    engine_args:
+      num_blocks: 2048
+      max_num_seqs: 8
+      max_num_batched_tokens: 16384
+      enable_prefix_caching: false
+    parameters:
+      model: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+    artifacts:
+      - name: model
+        target: model
+        source:
+          type: huggingface
+          repo: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+          revision: 19441ac874c4018295da848e250f23511361cda4
+- name: qwen3.8-27b-dflash2-exl3-vllm-cpp
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+    - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+    - https://github.com/mudler/vllm.cpp
+  description: |
+    Qwen3.8-27B EXL3 with its EXL3 DFlash2 companion, served by vllm.cpp. On
+    GB10 this pinned pair measured 48.7 tokens/s at a seven-token draft budget,
+    versus 16.7 tokens/s target-only, with token-identical greedy output.
+
+    LocalAI stages both complete Hugging Face repositories before load. The
+    content-addressed companion snapshot is passed to the backend as the draft
+    model, while the speculative method and seven-token budget remain fixed.
+  tags:
+    - llm
+    - qwen
+    - qwen3.8
+    - exl3
+    - vllm-cpp
+    - speculative-decoding
+    - dflash
+    - reasoning
+    - tool-calling
+    - gpu
+    - cuda
+  size: 16.9GB
+  last_checked: "2026-09-07"
+  variants:
+    - model: qwen3.8-27b-exl3-vllm-cpp
+  overrides:
+    backend: vllm-cpp
+    known_usecases:
+      - chat
+      - completion
+    function:
+      grammar:
+        disable: true
+    template:
+      use_tokenizer_template: true
+    context_size: 8192
+    engine_args:
+      num_blocks: 2048
+      max_num_seqs: 8
+      max_num_batched_tokens: 16384
+      enable_prefix_caching: false
+      speculative_config:
+        method: dflash
+        model: Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+        num_speculative_tokens: 7
+    parameters:
+      model: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+    artifacts:
+      - name: model
+        target: model
+        source:
+          type: huggingface
+          repo: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+          revision: 19441ac874c4018295da848e250f23511361cda4
+      - name: draft_model
+        target: companion
+        source:
+          type: huggingface
+          repo: Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+          revision: 4f0436269bca761b071f05319e8e04a87cc633f9
+- name: deepseek-v4-flash-spark-exl3-vllm-cpp
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/0xSero/deepseek-v4-flash-0731-spark
+    - https://github.com/mudler/vllm.cpp
+  description: |
+    DeepSeek V4 Flash's Spark and GB10-oriented REAP-K216 EXL3 checkpoint,
+    served by vllm.cpp. It needs CUDA and roughly 100 GiB for its large
+    rank-sliced checkpoint.
+
+    The repository is pinned to its latest recorded revision. vllm.cpp's
+    existing runtime evidence measured the older 22f28d32b9b29b4352eaa380ff8c2c170b2847ab
+    revision; this entry does not claim that the newer revision has passed the
+    same end-to-end gate.
+  tags:
+    - llm
+    - deepseek
+    - deepseek-v4
+    - exl3
+    - vllm-cpp
+    - gpu
+    - cuda
+    - gb10
+  size: 100GB
+  last_checked: "2026-09-07"
+  overrides:
+    backend: vllm-cpp
+    known_usecases:
+      - chat
+      - completion
+    template:
+      use_tokenizer_template: true
+    parameters:
+      model: 0xSero/deepseek-v4-flash-0731-spark
+    artifacts:
+      - name: model
+        target: model
+        source:
+          type: huggingface
+          repo: 0xSero/deepseek-v4-flash-0731-spark
+          revision: ce5ff0f1efb2e184aafc759d281bfae47d3a359c
+- name: deepseek-v4-flash-exl3-3bpw-vllm-cpp
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+    - https://github.com/mudler/vllm.cpp
+  description: |
+    Experimental non-Spark DeepSeek V4 Flash EXL3 3.0bpw checkpoint served by
+    vllm.cpp. This is the complete, non-REAP layout and requires a large
+    multi-GPU CUDA system.
+
+    The publisher describes the artifact as structurally complete but has not
+    passed end-to-end generation. Treat this entry as an integration target,
+    not as a correctness- or performance-gated configuration.
+  tags:
+    - llm
+    - deepseek
+    - deepseek-v4
+    - exl3
+    - vllm-cpp
+    - experimental
+    - gpu
+    - cuda
+  last_checked: "2026-09-07"
+  overrides:
+    backend: vllm-cpp
+    known_usecases:
+      - chat
+      - completion
+    template:
+      use_tokenizer_template: true
+    parameters:
+      model: 0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+    artifacts:
+      - name: model
+        target: model
+        source:
+          type: huggingface
+          repo: 0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+          revision: e0bf84ac76a5100e8790c22ad10b70b1e2d06d71
 - name: qwen3.6-27b-nvfp4-vllm-cpp
  url: github:mudler/LocalAI/gallery/virtual.yaml@master
  urls: