diff --git a/backend/go/vllm-cpp/backend.go b/backend/go/vllm-cpp/backend.go index 071b8f584..a53bf951d 100644 --- a/backend/go/vllm-cpp/backend.go +++ b/backend/go/vllm-cpp/backend.go @@ -109,6 +109,16 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error { v.opts = parseOptions(opts) + // A DFlash draft is a second checkpoint the engine opens by path, and the + // engine never downloads one. Resolve it against LocalAI's models directory + // now so a repo-id spelling works, and so a missing draft fails here with an + // actionable message rather than as an HF-cache miss inside the load. + resolvedSpec, err := resolveDraftModelPath(v.opts.speculativeConfig, opts.ModelPath) + if err != nil { + return err + } + v.opts.speculativeConfig = resolvedSpec + mp := defaultModelParams() if v.opts.blockSize > 0 { mp.BlockSize = v.opts.blockSize diff --git a/backend/go/vllm-cpp/options.go b/backend/go/vllm-cpp/options.go index b1fb2a4dd..b2d9a378e 100644 --- a/backend/go/vllm-cpp/options.go +++ b/backend/go/vllm-cpp/options.go @@ -18,6 +18,10 @@ package main import ( "encoding/json" + "fmt" + "os" + "path" + "path/filepath" "strconv" "strings" @@ -206,6 +210,76 @@ func jsonInt32(v any, fallback int32) int32 { } } +// resolveDraftModelPath rewrites a DFlash draft reference into an absolute path +// the engine can actually open. +// +// The engine resolves `speculative_config.model` against a directory containing +// config.json, or against ~/.cache/huggingface/hub/models----/ +// snapshots/* - and it NEVER downloads. LocalAI keeps models in its own +// directory, so a bare HF repo id (the spelling the vLLM docs teach) misses the +// HF cache and dies deep in the load with "draft checkpoint not found", which +// reads like a broken checkpoint rather than a missing download. +// +// So: try the reference as given, then the last path segment under the models +// dir (`z-lab/Qwen3.6-27B-DFlash` -> `/Qwen3.6-27B-DFlash`, which is +// what LocalAI's own downloader produces), then the whole reference under the +// models dir. If none exist, fail HERE with a message naming both what was +// asked for and where we looked. +// +// mtp and ngram carry no separate draft checkpoint, so they pass through. A +// document that does not parse also passes through: the engine owns config +// validation and produces the better error. +func resolveDraftModelPath(speculativeConfig, modelsDir string) (string, error) { + if strings.TrimSpace(speculativeConfig) == "" { + return speculativeConfig, nil + } + var spec map[string]any + if err := json.Unmarshal([]byte(speculativeConfig), &spec); err != nil { + return speculativeConfig, nil + } + if method, _ := spec["method"].(string); !strings.EqualFold(method, "dflash") { + return speculativeConfig, nil + } + + ref, _ := spec["model"].(string) + ref = strings.TrimSpace(ref) + if ref == "" { + return "", fmt.Errorf( + "vllm-cpp: speculative_config method %q requires a \"model\" key naming the draft checkpoint", "dflash") + } + + candidates := []string{ref} + if modelsDir != "" { + if base := path.Base(filepath.ToSlash(ref)); base != "" && base != "." && base != "/" { + candidates = append(candidates, filepath.Join(modelsDir, base)) + } + candidates = append(candidates, filepath.Join(modelsDir, filepath.FromSlash(ref))) + } + + for _, c := range candidates { + if _, err := os.Stat(filepath.Join(c, "config.json")); err != nil { + continue + } + abs, err := filepath.Abs(c) + if err != nil { + abs = c + } + spec["model"] = abs + out, err := json.Marshal(spec) + if err != nil { + return "", fmt.Errorf("vllm-cpp: re-encoding speculative_config: %w", err) + } + xlog.Info("[vllm-cpp] resolved DFlash draft checkpoint", "reference", ref, "path", abs) + return string(out), nil + } + + return "", fmt.Errorf( + "vllm-cpp: DFlash draft checkpoint %q not found (looked in: %s). "+ + "The engine does not download drafts - install the draft model into LocalAI first, "+ + "or set speculative_config.model to an absolute path to a directory containing config.json", + ref, strings.Join(candidates, ", ")) +} + func parseInt32(s string, fallback int32) int32 { n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 32) if err != nil || n <= 0 { diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 96112345d..126cff2c3 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -265,6 +265,91 @@ var _ = Describe("samplingFromPredict", func() { }) }) +// The engine resolves speculative_config.model against a local directory or +// ~/.cache/huggingface/hub ONLY - it never downloads. LocalAI keeps models in +// its own directory, so a bare repo id would miss the HF cache and fail deep in +// the load with a confusing "draft checkpoint not found". Resolve it here. +var _ = Describe("resolveDraftModelPath", func() { + var modelsDir string + + BeforeEach(func() { + modelsDir = GinkgoT().TempDir() + }) + + // draftDir creates a plausible draft checkpoint under models/. + draftDir := func(name string) string { + d := filepath.Join(modelsDir, name) + Expect(os.MkdirAll(d, 0o750)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(d, "config.json"), []byte("{}"), 0o600)).To(Succeed()) + return d + } + + It("rewrites a repo id to the matching directory in the models dir", func() { + want := draftDir("Qwen3.6-27B-DFlash") + spec := `{"method":"dflash","model":"z-lab/Qwen3.6-27B-DFlash"}` + out, err := resolveDraftModelPath(spec, modelsDir) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(MatchJSON(`{"method":"dflash","model":"` + want + `"}`)) + }) + + It("rewrites a models-dir-relative path", func() { + want := draftDir("drafts__dflash") + spec := `{"method":"dflash","model":"drafts__dflash"}` + out, err := resolveDraftModelPath(spec, modelsDir) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(ContainSubstring(want)) + }) + + It("leaves an absolute path that already resolves alone", func() { + abs := draftDir("elsewhere") + spec := `{"method":"dflash","model":"` + abs + `"}` + out, err := resolveDraftModelPath(spec, modelsDir) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(MatchJSON(spec)) + }) + + It("fails with an actionable error when the draft is nowhere on disk", func() { + // Silently passing the repo id through would surface as an HF-cache + // miss inside the engine, which reads as "your model is broken". + spec := `{"method":"dflash","model":"z-lab/Not-Downloaded"}` + _, err := resolveDraftModelPath(spec, modelsDir) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("z-lab/Not-Downloaded")) + Expect(err.Error()).To(ContainSubstring(modelsDir)) + }) + + It("requires a model key for dflash", func() { + _, err := resolveDraftModelPath(`{"method":"dflash"}`, modelsDir) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model")) + }) + + It("leaves mtp and ngram configs untouched", func() { + // Neither has a separate draft checkpoint to resolve. + for _, spec := range []string{ + `{"method":"mtp"}`, + `{"method":"ngram","num_speculative_tokens":4}`, + } { + out, err := resolveDraftModelPath(spec, modelsDir) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(MatchJSON(spec)) + } + }) + + It("passes a malformed document through for the engine to reject", func() { + // The engine owns config validation and produces the better message. + out, err := resolveDraftModelPath(`{not json`, modelsDir) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(Equal(`{not json`)) + }) + + It("is a no-op on an empty config", func() { + out, err := resolveDraftModelPath("", modelsDir) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(BeEmpty()) + }) +}) + var _ = Describe("validModelPath", func() { It("accepts a .gguf file", func() { dir := GinkgoT().TempDir() diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index 16060420e..8c44a1e4c 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -962,7 +962,22 @@ prefill from blowing up the per-step activation on the hybrid architectures. #### Speculative decoding `speculative_config:` takes the same JSON object as vLLM's -`--speculative-config`. Three methods are supported: +`--speculative-config`. Three methods are supported. + +> **Architecture limit.** At the current engine pin, `mtp` and `dflash` are +> **Qwen3.5 / Qwen3.6 only**. The engine builds a widened speculative KV cache +> directly for those families rather than through the model registry, so a +> speculative config on any other architecture (Llama, GLM, Gemma, Mistral, ...) +> will not work regardless of checkpoint format. `ngram` needs no draft weights +> and is not subject to this limit. + +> **Format limit.** `mtp` and `dflash` require a **safetensors** target and are +> rejected at load on a `.gguf` target, with: +> `speculative decoding requires a safetensors target checkpoint`. +> This is a current gap in the engine's GGUF loader, not a property of the GGUF +> format - GGUF can carry MTP weights (llama.cpp reads them as `nextn.*` +> tensors plus a `.nextn_predict_layers` key), but vllm.cpp's GGUF path +> does not map them yet. `ngram` works fine on GGUF. **MTP** (Multi-Token Prediction) uses a draft head shipped inside the target checkpoint's own `mtp.*` tensors, so there is no second model to download. It @@ -990,6 +1005,18 @@ engine_args: num_speculative_tokens: 4 ``` +The draft shares the *target's* `embed_tokens` and `lm_head`, so both must come +from the same model family and the target must be safetensors. + +**The engine does not download the draft.** `model:` is resolved, in order, +as a path as given, then as the last path segment under LocalAI's models +directory (`z-lab/Qwen3.6-27B-DFlash` → `/Qwen3.6-27B-DFlash`, which is +what LocalAI's own downloader produces), then as the whole reference under the +models directory. Install the draft into LocalAI first, or give an absolute path +to a directory containing `config.json`. If none of those resolve, the load +fails immediately naming every location that was tried, rather than reporting a +missing checkpoint from inside the engine. + **N-gram** needs no draft model at all - it proposes from the prompt's own suffix history. `num_speculative_tokens` is required: