Merge master into the distributed test branch

Include current gallery and virtual-model path fixes before continuing
the requested test repairs.

Assisted-by: Codex:gpt-6

backend/go/vllm-cpp/options.go

@@ -128,9 +128,40 @@ func parseOptions(opts *pb.ModelOptions) loadOptions {
 	lo := loadOptions{}
 	applyOptionsList(&lo, opts.GetOptions())
 	applyEngineArgs(&lo, opts.GetEngineArgs())
+	applyDraftModelOption(&lo, opts.GetOptions())
 	return lo
 }
 
+// applyDraftModelOption binds a managed companion snapshot after engine_args
+// has supplied the speculative document. Companion paths do not exist until
+// LocalAI materializes the artifact, so they must replace the gallery's static
+// repository reference without disturbing the method or token budget.
+func applyDraftModelOption(lo *loadOptions, options []string) {
+	if strings.TrimSpace(lo.speculativeConfig) == "" {
+		return
+	}
+	var draftModel string
+	for _, option := range options {
+		key, value, found := strings.Cut(option, ":")
+		if found && strings.TrimSpace(key) == "draft_model" {
+			draftModel = strings.TrimSpace(value)
+		}
+	}
+	if draftModel == "" {
+		return
+	}
+
+	var spec map[string]any
+	if err := json.Unmarshal([]byte(lo.speculativeConfig), &spec); err != nil {
+		return
+	}
+	spec["model"] = draftModel
+	encoded, err := json.Marshal(spec)
+	if err == nil {
+		lo.speculativeConfig = string(encoded)
+	}
+}
+
 // applyOptionsList reads the legacy free-form "key:value" list. strings.Cut
 // splits on the FIRST colon only, so a JSON object value survives intact.
 func applyOptionsList(lo *loadOptions, options []string) {

backend/go/vllm-cpp/options_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+)
+
+var _ = Describe("managed DFlash companion options", func() {
+	It("replaces only the draft model in an existing speculative configuration", func() {
+		managedPath := ".artifacts/huggingface/0123456789abcdef/snapshot"
+		lo := parseOptions(&pb.ModelOptions{
+			Options: []string{"draft_model:" + managedPath},
+			EngineArgs: `{
+				"speculative_config": {
+					"method": "dflash",
+					"model": "Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw",
+					"num_speculative_tokens": 7
+				}
+			}`,
+		})
+
+		Expect(lo.speculativeConfig).To(MatchJSON(`{
+			"method": "dflash",
+			"model": ".artifacts/huggingface/0123456789abcdef/snapshot",
+			"num_speculative_tokens": 7
+		}`))
+	})
+
+	It("ignores a draft companion when speculative decoding is not configured", func() {
+		lo := parseOptions(&pb.ModelOptions{
+			Options: []string{"draft_model:.artifacts/huggingface/0123456789abcdef/snapshot"},
+		})
+
+		Expect(lo.speculativeConfig).To(BeEmpty())
+	})
+})

core/gallery/exl3_gallery_test.go

@@ -0,0 +1,101 @@
+package gallery_test
+
+import (
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"gopkg.in/yaml.v3"
+)
+
+func remarshal(value any, target any) error {
+	data, err := yaml.Marshal(value)
+	if err != nil {
+		return err
+	}
+	return yaml.Unmarshal(data, target)
+}
+
+var _ = Describe("EXL3 gallery entries", func() {
+	It("pins the four full repositories and configures the Qwen DFlash companion", func() {
+		entries, err := gallery.ReadConfigFile[[]gallery.GalleryModel](filepath.Join("..", "..", "gallery", "index.yaml"))
+		Expect(err).ToNot(HaveOccurred())
+
+		byName := make(map[string]gallery.GalleryModel, len(*entries))
+		for _, entry := range *entries {
+			byName[entry.Name] = entry
+		}
+
+		expected := map[string]struct {
+			repo     string
+			revision string
+		}{
+			"qwen3.8-27b-exl3-vllm-cpp": {
+				repo: "Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw", revision: "19441ac874c4018295da848e250f23511361cda4",
+			},
+			"qwen3.8-27b-dflash2-exl3-vllm-cpp": {
+				repo: "Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw", revision: "19441ac874c4018295da848e250f23511361cda4",
+			},
+			"deepseek-v4-flash-spark-exl3-vllm-cpp": {
+				repo: "0xSero/deepseek-v4-flash-0731-spark", revision: "ce5ff0f1efb2e184aafc759d281bfae47d3a359c",
+			},
+			"deepseek-v4-flash-exl3-3bpw-vllm-cpp": {
+				repo: "0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw", revision: "e0bf84ac76a5100e8790c22ad10b70b1e2d06d71",
+			},
+		}
+
+		for name, want := range expected {
+			entry, found := byName[name]
+			Expect(found).To(BeTrue(), "missing gallery entry %q", name)
+			Expect(entry.Tags).To(ContainElements("vllm-cpp", "exl3", "gpu", "cuda"), name)
+			Expect(entry.Overrides).To(HaveKeyWithValue("backend", "vllm-cpp"), name)
+			cfg := config.ModelConfig{}
+			Expect(remarshal(entry.Overrides, &cfg)).To(Succeed(), name)
+			Expect(cfg.Artifacts).ToNot(BeEmpty(), name)
+			Expect(cfg.Artifacts[0].Source.Repo).To(Equal(want.repo), name)
+			Expect(cfg.Artifacts[0].Source.Revision).To(Equal(want.revision), name)
+		}
+
+		plain := byName["qwen3.8-27b-exl3-vllm-cpp"]
+		Expect(plain.Tags).ToNot(ContainElement("dflash"))
+		dflashTags := 0
+		for name := range expected {
+			if contains(byName[name].Tags, "dflash") {
+				dflashTags++
+			}
+		}
+		Expect(dflashTags).To(Equal(1))
+
+		dflash := byName["qwen3.8-27b-dflash2-exl3-vllm-cpp"]
+		Expect(dflash.Tags).To(ContainElement("dflash"))
+		Expect(dflash.Variants).To(ConsistOf(gallery.Variant{Model: "qwen3.8-27b-exl3-vllm-cpp"}))
+		cfg := config.ModelConfig{}
+		Expect(remarshal(dflash.Overrides, &cfg)).To(Succeed())
+		Expect(cfg.ContextSize).To(HaveValue(Equal(8192)))
+		Expect(cfg.EngineArgs).To(HaveKeyWithValue("num_blocks", 2048))
+		Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_seqs", 8))
+		Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_batched_tokens", 16384))
+		Expect(cfg.EngineArgs).To(HaveKeyWithValue("enable_prefix_caching", false))
+		spec, ok := cfg.EngineArgs["speculative_config"].(map[string]any)
+		Expect(ok).To(BeTrue())
+		Expect(spec).To(HaveKeyWithValue("method", "dflash"))
+		Expect(spec).To(HaveKeyWithValue("num_speculative_tokens", 7))
+		Expect(cfg.Artifacts).To(HaveLen(2))
+		Expect(cfg.Artifacts[1].Name).To(Equal("draft_model"))
+		Expect(cfg.Artifacts[1].Target).To(Equal("companion"))
+		Expect(cfg.Artifacts[1].Source.Repo).To(Equal("Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw"))
+		Expect(cfg.Artifacts[1].Source.Revision).To(Equal("4f0436269bca761b071f05319e8e04a87cc633f9"))
+	})
+})
+
+func contains(values []string, target string) bool {
+	for _, value := range values {
+		if value == target {
+			return true
+		}
+	}
+	return false
+}

core/services/nodes/router.go

@@ -1729,8 +1729,15 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
 
 	// Stage file paths referenced in generic Options (key:value pairs where values
 	// are file paths). Options stay as relative paths — backends resolve them via ModelPath.
-	r.stageGenericOptions(ctx, node, opts.Options, frontendModelsDir, localModelDir, keyMapper.Key)
-	r.stageGenericOptions(ctx, node, opts.Overrides, frontendModelsDir, localModelDir, keyMapper.Key)
+	for _, options := range [][]string{opts.Options, opts.Overrides} {
+		remoteRoot := r.stageGenericOptions(ctx, node, options, frontendModelsDir, localModelDir, keyMapper.Key)
+		if opts.ModelFile == "" && remoteRoot != "" {
+			// Virtual models have no primary file from which to derive the
+			// worker root. Their relative options must resolve against the
+			// companion assets we actually staged, not the frontend's root.
+			opts.ModelPath = remoteRoot
+		}
+	}
 
 	return opts, nil
 }
@@ -1961,7 +1968,9 @@ func (r *SmartRouter) stageCompanionFiles(ctx context.Context, node *BackendNode
 // that resolve to existing files relative to the frontend models directory or
 // the model's own directory. Option values are NOT rewritten — backends resolve
 // them via ModelPath. keyFn generates the namespaced storage key for each file.
-func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode, options []string, frontendModelsDir, modelDir string, keyFn func(string) string) {
+// Returns the staged models root, or empty when no asset was staged.
+func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode, options []string, frontendModelsDir, modelDir string, keyFn func(string) string) string {
+	remoteRoot := ""
 	for _, opt := range options {
 		optKey, val, ok := strings.Cut(opt, ":")
 		if !ok || val == "" {
@@ -1986,18 +1995,23 @@ func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode
 		// worker; a single file is staged directly. Values are never rewritten —
 		// backends resolve relative paths via ModelPath.
 		if err == nil && info.IsDir() {
-			r.stageOptionDir(ctx, node, absPath, keyFn)
+			if remoteDir := r.stageOptionDir(ctx, node, absPath, keyFn); remoteDir != "" {
+				remoteRoot = DeriveRemoteModelPath(remoteDir, relativeToModelsDir(frontendModelsDir, absPath, filepath.Base(absPath)))
+			}
 			xlog.Debug("Staged option directory", "option", optKey, "localPath", absPath)
 			continue
 		}
 
 		key := keyFn(absPath)
-		if _, err := r.fileStager.EnsureRemote(ctx, node.ID, absPath, key); err != nil {
+		remotePath, err := r.fileStager.EnsureRemote(ctx, node.ID, absPath, key)
+		if err != nil {
 			xlog.Warn("Failed to stage option file, skipping", "option", opt, "path", absPath, "error", err)
 			continue
 		}
+		remoteRoot = DeriveRemoteModelPath(remotePath, relativeToModelsDir(frontendModelsDir, absPath, filepath.Base(absPath)))
 		xlog.Debug("Staged option file", "option", optKey, "localPath", absPath)
 	}
+	return remoteRoot
 }
 
 // resolveOptionPath finds an existing local path for an option value: an
@@ -2025,8 +2039,10 @@ func resolveOptionPath(val, frontendModelsDir, modelDir string) (string, bool) {
 // stageOptionDir stages every regular file under an option-declared directory
 // (e.g. sherpa-onnx's espeak-ng-data) using the structure-preserving key, so the
 // tree is recreated beside the model on the worker. Per-file errors are logged
-// and skipped; the option value itself is not rewritten.
-func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir string, keyFn func(string) string) {
+// and skipped; the option value itself is not rewritten. Returns the remote
+// directory derived from a successfully staged file, or empty when none succeeds.
+func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir string, keyFn func(string) string) string {
+	remoteDir := ""
 	_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
 		if walkErr != nil || d.IsDir() {
 			return nil
@@ -2041,11 +2057,17 @@ func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir
 		if isHashSidecar(path) {
 			return nil
 		}
-		if _, err := r.fileStager.EnsureRemote(ctx, node.ID, path, keyFn(path)); err != nil {
+		remotePath, err := r.fileStager.EnsureRemote(ctx, node.ID, path, keyFn(path))
+		if err != nil {
 			xlog.Warn("Failed to stage option directory file, skipping", "path", path, "error", err)
+			return nil
+		}
+		if rel, err := filepath.Rel(dir, path); err == nil {
+			remoteDir = DeriveRemoteModelPath(remotePath, rel)
 		}
 		return nil
 	})
+	return remoteDir
 }
 
 // probeHealth checks whether a backend process on the given node/addr is alive

core/services/nodes/router_virtual_model_test.go

@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: MIT
+
+package nodes
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+type failedCompanionStager struct{ FileStager }
+
+func (failedCompanionStager) EnsureRemote(context.Context, string, string, string) (string, error) {
+	return "", errors.New("worker unavailable")
+}
+
+var _ = Describe("staging virtual model companions", func() {
+	DescribeTable("anchors relative assets on the worker",
+		func(options, overrides []string, files []string) {
+			modelsDir := GinkgoT().TempDir()
+			for _, name := range files {
+				path := filepath.Join(modelsDir, name)
+				Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+				Expect(os.WriteFile(path, []byte("weights"), 0600)).To(Succeed())
+			}
+			stager := &fakeFileStager{}
+			router := &SmartRouter{fileStager: stager, stagingTracker: NewStagingTracker()}
+			input := &pb.ModelOptions{Model: "insightface-buffalo-m", ModelFile: filepath.Join(modelsDir, "insightface-buffalo-m"), ModelPath: modelsDir, Options: options, Overrides: overrides}
+			staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "insightface-buffalo-m")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(staged.ModelPath).To(Equal("/remote/models/insightface-buffalo-m"))
+			Expect(staged.Options).To(Equal(options))
+			Expect(staged.Overrides).To(Equal(overrides))
+			Expect(input.ModelPath).To(Equal(modelsDir))
+			Expect(input.ModelFile).To(Equal(filepath.Join(modelsDir, "insightface-buffalo-m")))
+			Expect(stager.ensureCalls).To(HaveLen(len(files)))
+			for _, call := range stager.ensureCalls {
+				rel, err := filepath.Rel(modelsDir, call.localPath)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(filepath.Join(staged.ModelPath, rel)).To(Equal("/remote/" + call.key))
+			}
+		},
+		Entry("Buffalo pack and MiniFASNet files", []string{"model_pack:buffalo_m", "antispoof_v2_onnx:MiniFASNetV2.onnx", "antispoof_v1se_onnx:MiniFASNetV1SE.onnx"}, nil, []string{"buffalo_m/det_2.5g.onnx", "buffalo_m/w600k_r50.onnx", "MiniFASNetV2.onnx", "MiniFASNetV1SE.onnx"}),
+		Entry("only a nested companion directory", []string{"model_pack:packs/buffalo_m"}, nil, []string{"packs/buffalo_m/det_2.5g.onnx"}),
+		Entry("only a companion file", []string{"antispoof_v2_onnx:MiniFASNetV2.onnx"}, nil, []string{"MiniFASNetV2.onnx"}),
+		Entry("only override assets", nil, []string{"antispoof_v2_onnx:MiniFASNetV2.onnx"}, []string{"MiniFASNetV2.onnx"}),
+	)
+	It("keeps the original path when no assets are staged", func() {
+		modelsDir := GinkgoT().TempDir()
+		router := &SmartRouter{fileStager: &fakeFileStager{}, stagingTracker: NewStagingTracker()}
+		input := &pb.ModelOptions{Model: "virtual", ModelFile: filepath.Join(modelsDir, "virtual"), ModelPath: modelsDir, Options: []string{"engine:insightface"}}
+		staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "virtual")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(staged.ModelPath).To(Equal(modelsDir))
+	})
+	DescribeTable("keeps the original root when companion staging fails",
+		func(directory bool) {
+			modelsDir := GinkgoT().TempDir()
+			relative := "MiniFASNetV2.onnx"
+			if directory {
+				relative = "buffalo_m/det_2.5g.onnx"
+			}
+			local := filepath.Join(modelsDir, relative)
+			Expect(os.MkdirAll(filepath.Dir(local), 0750)).To(Succeed())
+			Expect(os.WriteFile(local, []byte("weights"), 0600)).To(Succeed())
+			value := relative
+			if directory {
+				value = "buffalo_m"
+			}
+			router := &SmartRouter{fileStager: failedCompanionStager{}, stagingTracker: NewStagingTracker()}
+			input := &pb.ModelOptions{Model: "virtual", ModelFile: filepath.Join(modelsDir, "virtual"), ModelPath: modelsDir, Options: []string{"asset:" + value}}
+			staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "virtual")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(staged.ModelPath).To(Equal(modelsDir))
+		}, Entry("file", false), Entry("directory", true),
+	)
+
+})

docs/content/features/distributed-mode.md

@@ -1739,6 +1739,10 @@ Notes:
 - Verify `--heartbeat-interval` is not set too high
 - Offline nodes automatically restore to healthy when they re-register (no re-approval needed)
 
+**InsightFace reports a missing MiniFASNet file after staging:**
+- Gallery models such as `insightface-buffalo-m` use a virtual primary name and load their files through options. The frontend derives the worker's model directory from successfully staged companion files or directories, so relative options resolve inside the model's staging directory.
+- If logs show matching hashes for the staged files but InsightFace still reports a bare filename such as `MiniFASNetV2.onnx` as missing, upgrade the frontend to include this path-resolution fix. Re-uploading the same files does not correct the directory passed to the backend.
+
 **Backend not installing:**
 - Check the worker logs for `backend.install` events
 

docs/superpowers/specs/2026-09-07-exl3-gallery-design.md

@@ -0,0 +1,99 @@
+# EXL3 gallery entries
+
+## Goal
+
+Add four gallery entries that expose the EXL3 configurations validated or
+tracked by `vllm.cpp`. Pin each Hugging Face artifact to the revision recorded
+by its source or benchmark evidence.
+
+## Entries
+
+### Qwen3.8 target
+
+Add `qwen3.8-27b-exl3-vllm-cpp` for
+`Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw`. This entry serves the target without a
+draft model.
+
+Use revision `19441ac874c4018295da848e250f23511361cda4`. Configure an 8,192-token
+context, 2,048 cache blocks, eight sequences, and 16,384 batched tokens. Disable
+prefix caching to match the measured serving configuration.
+
+### Qwen3.8 with DFlash2
+
+Add `qwen3.8-27b-dflash2-exl3-vllm-cpp`. This entry stages the Qwen3.8 target
+and `Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw` at revision
+`4f0436269bca761b071f05319e8e04a87cc633f9`.
+
+Configure the `dflash` method with seven speculative tokens. Use the shipped
+paged draft route. Apply the same serving limits as the target-only entry.
+
+Tag this entry with `dflash` because it enables speculative decoding. Declare
+the target-only entry as its variant. LocalAI can then prefer the faster entry
+when the host supports it.
+
+### DeepSeek V4 Flash for Spark
+
+Add `deepseek-v4-flash-spark-exl3-vllm-cpp` for
+`0xSero/deepseek-v4-flash-0731-spark`. Use the current repository revision,
+`ce5ff0f1efb2e184aafc759d281bfae47d3a359c`. State that the `vllm.cpp`
+runtime record used the older revision `22f28d32b9b29b4352eaa380ff8c2c170b2847ab`.
+
+Describe the entry as a Spark and GB10-oriented REAP-K216 checkpoint. State its
+large memory requirement and CUDA requirement. Do not claim a completed speed
+or correctness gate that the source record does not contain.
+
+### DeepSeek V4 Flash 3.0 bpw
+
+Add `deepseek-v4-flash-exl3-3bpw-vllm-cpp` for
+`0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw`. Use the current repository
+revision, `e0bf84ac76a5100e8790c22ad10b70b1e2d06d71`.
+
+Tag and describe this entry as experimental. The model card states that the
+artifact is structurally complete, but end-to-end generation has not passed.
+Keep this entry separate from the Spark entry because the repositories use
+different layouts and have different runtime evidence.
+
+## Artifact staging
+
+Use LocalAI's Hugging Face artifact source for each repository. Stage complete
+model repositories because these safetensors checkpoints need configuration,
+tokenizer, index, and weight files.
+
+Assign the Qwen draft artifact to a companion target. Pass its staged path in
+the `vllm-cpp` speculative configuration. Do not download files through backend
+startup logic.
+
+## User-visible metadata
+
+Use the `vllm-cpp`, `exl3`, `gpu`, and `cuda` tags on all four entries. Add
+architecture, reasoning, tool-calling, and speculative-decoding tags only when
+the configured model supports them.
+
+Descriptions must distinguish measured results from unresolved work. The Qwen
+DFlash2 description can cite the measured configuration and throughput. The
+DeepSeek descriptions must not imply an end-to-end validation that does not
+exist.
+
+## Validation
+
+Run the gallery schema and focused gallery tests. Add a focused test if the
+artifact or variant structure is not already covered.
+
+Validate these properties:
+
+- Every name is unique.
+- Every variant points to an existing entry.
+- Each Hugging Face source has a pinned revision.
+- The DFlash2 entry stages both repositories and passes the draft path.
+- Only the configured DFlash2 entry has the `dflash` tag.
+- YAML parsing and gallery loading succeed.
+
+No model download or GPU benchmark is part of this LocalAI change. The
+`vllm.cpp` evidence supplies the runtime record.
+
+## Out of scope
+
+- Changes to the `vllm-cpp` backend binaries.
+- New EXL3 kernels or model loaders.
+- New benchmark claims.
+- Gallery entries for unselected EXL3 bit widths.

gallery/index.yaml

@@ -14484,6 +14484,202 @@
             - vae/**
     parameters:
       model: meituan-longcat/LongCat-Video-Avatar-1.5
+- name: qwen3.8-27b-exl3-vllm-cpp
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+    - https://github.com/mudler/vllm.cpp
+  description: |
+    Qwen3.8-27B EXL3 3.5bpw served by vllm.cpp, LocalAI's C++ vLLM-style
+    runtime. The published checkpoint generates on CUDA and measured 16.7
+    tokens/s on GB10 with the pinned revision and the limits configured here.
+
+    This is the target-only setup. Use the DFlash2 variant for the measured
+    speculative-decoding configuration. The entry downloads the complete
+    revision-pinned repository, including its configuration, tokenizer, index,
+    and safetensors shards.
+  tags:
+    - llm
+    - qwen
+    - qwen3.8
+    - exl3
+    - vllm-cpp
+    - reasoning
+    - tool-calling
+    - gpu
+    - cuda
+  size: 15.4GB
+  last_checked: "2026-09-07"
+  overrides:
+    backend: vllm-cpp
+    known_usecases:
+      - chat
+      - completion
+    function:
+      grammar:
+        disable: true
+    template:
+      use_tokenizer_template: true
+    context_size: 8192
+    engine_args:
+      num_blocks: 2048
+      max_num_seqs: 8
+      max_num_batched_tokens: 16384
+      enable_prefix_caching: false
+    parameters:
+      model: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+    artifacts:
+      - name: model
+        target: model
+        source:
+          type: huggingface
+          repo: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+          revision: 19441ac874c4018295da848e250f23511361cda4
+- name: qwen3.8-27b-dflash2-exl3-vllm-cpp
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+    - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+    - https://github.com/mudler/vllm.cpp
+  description: |
+    Qwen3.8-27B EXL3 with its EXL3 DFlash2 companion, served by vllm.cpp. On
+    GB10 this pinned pair measured 48.7 tokens/s at a seven-token draft budget,
+    versus 16.7 tokens/s target-only, with token-identical greedy output.
+
+    LocalAI stages both complete Hugging Face repositories before load. The
+    content-addressed companion snapshot is passed to the backend as the draft
+    model, while the speculative method and seven-token budget remain fixed.
+  tags:
+    - llm
+    - qwen
+    - qwen3.8
+    - exl3
+    - vllm-cpp
+    - speculative-decoding
+    - dflash
+    - reasoning
+    - tool-calling
+    - gpu
+    - cuda
+  size: 16.9GB
+  last_checked: "2026-09-07"
+  variants:
+    - model: qwen3.8-27b-exl3-vllm-cpp
+  overrides:
+    backend: vllm-cpp
+    known_usecases:
+      - chat
+      - completion
+    function:
+      grammar:
+        disable: true
+    template:
+      use_tokenizer_template: true
+    context_size: 8192
+    engine_args:
+      num_blocks: 2048
+      max_num_seqs: 8
+      max_num_batched_tokens: 16384
+      enable_prefix_caching: false
+      speculative_config:
+        method: dflash
+        model: Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+        num_speculative_tokens: 7
+    parameters:
+      model: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+    artifacts:
+      - name: model
+        target: model
+        source:
+          type: huggingface
+          repo: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+          revision: 19441ac874c4018295da848e250f23511361cda4
+      - name: draft_model
+        target: companion
+        source:
+          type: huggingface
+          repo: Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+          revision: 4f0436269bca761b071f05319e8e04a87cc633f9
+- name: deepseek-v4-flash-spark-exl3-vllm-cpp
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/0xSero/deepseek-v4-flash-0731-spark
+    - https://github.com/mudler/vllm.cpp
+  description: |
+    DeepSeek V4 Flash's Spark and GB10-oriented REAP-K216 EXL3 checkpoint,
+    served by vllm.cpp. It needs CUDA and roughly 100 GiB for its large
+    rank-sliced checkpoint.
+
+    The repository is pinned to its latest recorded revision. vllm.cpp's
+    existing runtime evidence measured the older 22f28d32b9b29b4352eaa380ff8c2c170b2847ab
+    revision; this entry does not claim that the newer revision has passed the
+    same end-to-end gate.
+  tags:
+    - llm
+    - deepseek
+    - deepseek-v4
+    - exl3
+    - vllm-cpp
+    - gpu
+    - cuda
+    - gb10
+  size: 100GB
+  last_checked: "2026-09-07"
+  overrides:
+    backend: vllm-cpp
+    known_usecases:
+      - chat
+      - completion
+    template:
+      use_tokenizer_template: true
+    parameters:
+      model: 0xSero/deepseek-v4-flash-0731-spark
+    artifacts:
+      - name: model
+        target: model
+        source:
+          type: huggingface
+          repo: 0xSero/deepseek-v4-flash-0731-spark
+          revision: ce5ff0f1efb2e184aafc759d281bfae47d3a359c
+- name: deepseek-v4-flash-exl3-3bpw-vllm-cpp
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+    - https://github.com/mudler/vllm.cpp
+  description: |
+    Experimental non-Spark DeepSeek V4 Flash EXL3 3.0bpw checkpoint served by
+    vllm.cpp. This is the complete, non-REAP layout and requires a large
+    multi-GPU CUDA system.
+
+    The publisher describes the artifact as structurally complete but has not
+    passed end-to-end generation. Treat this entry as an integration target,
+    not as a correctness- or performance-gated configuration.
+  tags:
+    - llm
+    - deepseek
+    - deepseek-v4
+    - exl3
+    - vllm-cpp
+    - experimental
+    - gpu
+    - cuda
+  last_checked: "2026-09-07"
+  overrides:
+    backend: vllm-cpp
+    known_usecases:
+      - chat
+      - completion
+    template:
+      use_tokenizer_template: true
+    parameters:
+      model: 0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+    artifacts:
+      - name: model
+        target: model
+        source:
+          type: huggingface
+          repo: 0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+          revision: e0bf84ac76a5100e8790c22ad10b70b1e2d06d71
 - name: qwen3.6-27b-nvfp4-vllm-cpp
   url: github:mudler/LocalAI/gallery/virtual.yaml@master
   urls: