diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 0f009340c..4eb82601e 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=9e1c9025ae61167a3335454d7cc0de6093c21845 +VLLM_CPP_VERSION?=974d9d72c2365d20dc76dbce4a98fddac32525c0 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the # server, examples and tests of the engine are never built here. diff --git a/backend/go/vllm-cpp/backend.go b/backend/go/vllm-cpp/backend.go index 63cb79485..071b8f584 100644 --- a/backend/go/vllm-cpp/backend.go +++ b/backend/go/vllm-cpp/backend.go @@ -116,34 +116,60 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error { if v.opts.numBlocks > 0 { mp.NumBlocks = v.opts.numBlocks } + // Sequence-length precedence, narrowest source last: context_size is the + // generic LocalAI knob every backend honours, max_model_len is the + // vLLM-specific one, and engine_args.max_model_len is the explicit + // vllm-cpp override. if opts.ContextSize > 0 { mp.MaxModelLen = opts.ContextSize } + if opts.MaxModelLen > 0 { + mp.MaxModelLen = opts.MaxModelLen + } + if v.opts.maxModelLen > 0 { + mp.MaxModelLen = v.opts.maxModelLen + } if v.opts.maxNumSeqs > 0 { mp.MaxNumSeqs = v.opts.maxNumSeqs } + if v.opts.maxNumBatchedTokens > 0 { + mp.MaxNumBatchedTokens = v.opts.maxNumBatchedTokens + } + mp.EnablePrefixCaching = v.opts.enablePrefixCaching + // Every string below is borrowed by C for the duration of the load call + // only (the library copies what it keeps), so the backing slices just have + // to outlive vllmEngineLoad - hence the single KeepAlive after it. modelC := cString(model) mp.ModelPath = uintptr(unsafe.Pointer(&modelC[0])) // #nosec G103 -- borrowed by C for the load call only - var toolParserC, reasoningParserC []byte - if v.opts.toolParser != "" { - toolParserC = cString(v.opts.toolParser) - mp.ToolParser = uintptr(unsafe.Pointer(&toolParserC[0])) // #nosec G103 -- borrowed by C for the load call only - } - if v.opts.reasoningParser != "" { - reasoningParserC = cString(v.opts.reasoningParser) - mp.ReasoningParser = uintptr(unsafe.Pointer(&reasoningParserC[0])) // #nosec G103 -- borrowed by C for the load call only + keep := [][]byte{modelC} + setStr := func(dst *uintptr, s string) { + if s == "" { + return + } + b := cString(s) + keep = append(keep, b) + *dst = uintptr(unsafe.Pointer(&b[0])) // #nosec G103 -- borrowed by C for the load call only } + setStr(&mp.ToolParser, v.opts.toolParser) + setStr(&mp.ReasoningParser, v.opts.reasoningParser) + setStr(&mp.SpeculativeConfig, v.opts.speculativeConfig) + setStr(&mp.KVTransferConfig, v.opts.kvTransferConfig) + setStr(&mp.SchedulingPolicy, v.opts.schedulingPolicy) + setStr(&mp.TokenizerConfigPath, v.opts.tokenizerConfigPath) xlog.Info("[vllm-cpp] Load", "model", model, "engine", vllmVersion(), "blockSize", mp.BlockSize, "numBlocks", mp.NumBlocks, - "maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs) + "maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs, + "maxNumBatchedTokens", mp.MaxNumBatchedTokens, + "prefixCaching", prefixCachingName(mp.EnablePrefixCaching), + "schedulingPolicy", v.opts.schedulingPolicy, + "speculativeConfig", v.opts.speculativeConfig, + "kvTransferConfig", v.opts.kvTransferConfig) var engine uintptr rc := vllmEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params - runtime.KeepAlive(modelC) - runtime.KeepAlive(toolParserC) - runtime.KeepAlive(reasoningParserC) + runtime.KeepAlive(keep) if rc != vllmOK { return fmt.Errorf("vllm-cpp: engine load failed: %s", vllmLastError()) } diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index bd39e5428..c601f139e 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v2). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v9). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -18,23 +18,52 @@ import ( ) // abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h). -const abiVersion = 5 +const abiVersion = 9 + +// Prefix-caching tri-state (vllm_model_params.enable_prefix_caching, ABI v7). +// 0 is the model-capability default, which is what a config that says nothing +// about prefix caching must produce. +const ( + prefixCachingModelDefault int32 = 0 + prefixCachingOn int32 = 1 + prefixCachingOff int32 = 2 +) + +// prefixCachingName renders the tri-state for the load log line, where "0" +// would otherwise read as "off" rather than "whatever the model defaults to". +func prefixCachingName(state int32) string { + switch state { + case prefixCachingOn: + return "on" + case prefixCachingOff: + return "off" + default: + return "model-default" + } +} // vllm_status (vllm.h). const ( vllmOK = 0 ) -// cModelParams mirrors vllm_model_params. +// cModelParams mirrors vllm_model_params. Every field is naturally aligned on +// LP64 (the int32 pairs sit together), so the Go layout needs no explicit +// padding; the offsets are asserted in vllmcpp_test.go. type cModelParams struct { ModelPath uintptr // const char* - TokenizerConfigPath uintptr // const char* + TokenizerConfigPath uintptr // const char*; NULL = /... (ABI v9) BlockSize int32 NumBlocks int32 MaxModelLen int32 MaxNumSeqs int32 ToolParser uintptr // const char*; NULL = auto-detect (ABI v4) ReasoningParser uintptr // const char*; NULL = auto-detect (ABI v5) + SpeculativeConfig uintptr // const char* JSON; NULL = no speculation (ABI v6) + EnablePrefixCaching int32 // tri-state 0/1/2 (ABI v7) + MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9) + SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) + KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) } // cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields @@ -65,6 +94,12 @@ type cSamplingParams struct { StructuredGrammar uintptr // const char* StructuredJSONObject int32 _ [4]byte + // ABI v8 tail. LocalAI installs no custom logits processor, but the fields + // MUST be mirrored: the C side reads them off the pointer we hand it, so a + // Go struct that stopped at StructuredJSONObject would have the engine read + // 16 bytes past our allocation and call whatever garbage sat there. + LogitsProcessor uintptr // vllm_logits_processor; NULL = none + LogitsProcessorUserData uintptr // void* } // cCompletion mirrors vllm_completion. diff --git a/backend/go/vllm-cpp/options.go b/backend/go/vllm-cpp/options.go index 549f504c3..b1fb2a4dd 100644 --- a/backend/go/vllm-cpp/options.go +++ b/backend/go/vllm-cpp/options.go @@ -1,30 +1,72 @@ package main -// Engine-sizing knobs carried through the model config's free-form -// `options:` list ("key:value" entries), mirroring how the other in-house -// backends pass engine-specific settings that have no proto field. +// Load-time engine configuration, from two config surfaces: +// +// - `engine_args:` (ModelOptions.EngineArgs, a JSON object) is the canonical +// one. Keys are spelled exactly as vLLM's own CLI flags, so a config written +// against vLLM works verbatim here - `speculative_config` and +// `kv_transfer_config` in particular take the same JSON documents vLLM's +// --speculative-config / --kv-transfer-config accept, and are handed to the +// engine unparsed. +// - `options:` (the free-form "key:value" list) is the older surface this +// backend shipped with. It is still honoured so existing configs keep +// working; engine_args wins on any key set in both. +// +// Anything unrecognised is ignored rather than fatal: the engine validates the +// documents it is given and reports a precise error at load, and a config that +// also carries knobs for a different backend must not fail the load here. import ( + "encoding/json" "strconv" "strings" pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" ) type loadOptions struct { blockSize int32 // KV block size (tokens/block); engine default 32. numBlocks int32 // KV blocks to allocate; engine default 256. maxNumSeqs int32 // max concurrent sequences; engine default 8. + // Max sequence length. Also settable through the model config's + // context_size / max_model_len; see Load for the precedence. + maxModelLen int32 + // Per-step chunked-prefill token budget (ABI v9). 0 = the engine's + // bounded per-arch default. + maxNumBatchedTokens int32 + // Automatic prefix caching tri-state (ABI v7): 0 = the model-capability + // default, 1 = force on, 2 = force off. + enablePrefixCaching int32 + // Scheduler admission policy (ABI v9): "" = fcfs, else fcfs|priority|lpm. + schedulingPolicy string // Engine-side parser selection (ABI v4/v5). Empty = the engine // auto-detects from the chat template; "none" disables the reasoning // split; unknown names fail the first chat call. toolParser string reasoningParser string + // Speculative decoding (ABI v6), as vLLM's --speculative-config JSON: + // {"method":"mtp"|"dflash"|"ngram", ...}. Empty = no speculation. + speculativeConfig string + // External KV connector / LMCache (ABI v9), as vLLM's --kv-transfer-config + // JSON. Empty = no connector. + kvTransferConfig string + // Override for the tokenizer_config.json the chat template is read from + // (ABI v9). Empty = /tokenizer_config.json. + tokenizerConfigPath string } func parseOptions(opts *pb.ModelOptions) loadOptions { lo := loadOptions{} - for _, o := range opts.GetOptions() { + applyOptionsList(&lo, opts.GetOptions()) + applyEngineArgs(&lo, opts.GetEngineArgs()) + return lo +} + +// applyOptionsList reads the legacy free-form "key:value" list. strings.Cut +// splits on the FIRST colon only, so a JSON object value survives intact. +func applyOptionsList(lo *loadOptions, options []string) { + for _, o := range options { k, v, found := strings.Cut(o, ":") if !found { continue @@ -36,13 +78,132 @@ func parseOptions(opts *pb.ModelOptions) loadOptions { lo.numBlocks = parseInt32(v, lo.numBlocks) case "max_num_seqs": lo.maxNumSeqs = parseInt32(v, lo.maxNumSeqs) - case "tool_parser": + case "max_num_batched_tokens": + lo.maxNumBatchedTokens = parseInt32(v, lo.maxNumBatchedTokens) + case "max_model_len": + lo.maxModelLen = parseInt32(v, lo.maxModelLen) + case "scheduling_policy", "schedule_policy": + lo.schedulingPolicy = strings.TrimSpace(v) + case "tool_parser", "tool_call_parser": lo.toolParser = strings.TrimSpace(v) case "reasoning_parser": lo.reasoningParser = strings.TrimSpace(v) + case "speculative_config": + lo.speculativeConfig = strings.TrimSpace(v) + case "kv_transfer_config": + lo.kvTransferConfig = strings.TrimSpace(v) + case "tokenizer_config", "tokenizer_config_path": + lo.tokenizerConfigPath = strings.TrimSpace(v) + case "enable_prefix_caching", "enable_radix_attention": + if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil { + lo.enablePrefixCaching = prefixCachingTriState(b) + } } } - return lo +} + +// applyEngineArgs overlays the `engine_args:` JSON object. A document that does +// not parse is logged and skipped: engine_args is shared with the other engines +// (the vLLM and SGLang backends read the same field), so a stray key must not +// take the model down. +func applyEngineArgs(lo *loadOptions, engineArgs string) { + if strings.TrimSpace(engineArgs) == "" { + return + } + var args map[string]any + if err := json.Unmarshal([]byte(engineArgs), &args); err != nil { + xlog.Warn("[vllm-cpp] ignoring unparseable engine_args", "error", err) + return + } + for k, v := range args { + switch k { + case "block_size": + lo.blockSize = jsonInt32(v, lo.blockSize) + case "num_blocks": + lo.numBlocks = jsonInt32(v, lo.numBlocks) + case "max_num_seqs": + lo.maxNumSeqs = jsonInt32(v, lo.maxNumSeqs) + case "max_num_batched_tokens": + lo.maxNumBatchedTokens = jsonInt32(v, lo.maxNumBatchedTokens) + case "max_model_len": + lo.maxModelLen = jsonInt32(v, lo.maxModelLen) + case "scheduling_policy", "schedule_policy": + lo.schedulingPolicy = jsonString(v, lo.schedulingPolicy) + case "tool_parser", "tool_call_parser": + lo.toolParser = jsonString(v, lo.toolParser) + case "reasoning_parser": + lo.reasoningParser = jsonString(v, lo.reasoningParser) + case "tokenizer_config", "tokenizer_config_path": + lo.tokenizerConfigPath = jsonString(v, lo.tokenizerConfigPath) + case "speculative_config": + lo.speculativeConfig = jsonDocument(v, lo.speculativeConfig, k) + case "kv_transfer_config": + lo.kvTransferConfig = jsonDocument(v, lo.kvTransferConfig, k) + case "enable_prefix_caching", "enable_radix_attention": + if b, ok := v.(bool); ok { + lo.enablePrefixCaching = prefixCachingTriState(b) + } + default: + xlog.Debug("[vllm-cpp] ignoring unknown engine_args key", "key", k) + } + } +} + +// prefixCachingTriState maps a YAML/JSON boolean onto the C ABI's tri-state. An +// explicit `false` must reach the engine as force-OFF (2), NOT as the 0 that +// means "let the model capability decide" - those differ for the hybrid archs, +// which default the cache off, and for the dense ones, which default it on. +func prefixCachingTriState(on bool) int32 { + if on { + return prefixCachingOn + } + return prefixCachingOff +} + +// jsonDocument normalises an object-valued engine_args entry to a JSON string +// for the C ABI. YAML nesting arrives as a map (the natural spelling); a +// pre-encoded JSON string is accepted too, since a config round-tripped through +// a flat store may carry it that way. +func jsonDocument(v any, fallback string, key string) string { + switch t := v.(type) { + case string: + if strings.TrimSpace(t) == "" { + return fallback + } + return t + default: + buf, err := json.Marshal(t) + if err != nil { + xlog.Warn("[vllm-cpp] ignoring unencodable engine_args value", "key", key, "error", err) + return fallback + } + return string(buf) + } +} + +func jsonString(v any, fallback string) string { + s, ok := v.(string) + if !ok { + return fallback + } + return strings.TrimSpace(s) +} + +// jsonInt32 accepts the float64 a JSON number decodes to, plus the string +// spelling a YAML config may produce. Non-positive values keep the fallback: +// every knob this covers uses "<= 0 means the engine default". +func jsonInt32(v any, fallback int32) int32 { + switch t := v.(type) { + case float64: + if t <= 0 || t > 1<<31-1 { + return fallback + } + return int32(t) + case string: + return parseInt32(t, fallback) + default: + return fallback + } } func parseInt32(s string, fallback int32) int32 { diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 297d15944..69e55de59 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v2) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v9) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -30,10 +30,15 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.MaxNumSeqs)).To(Equal(uintptr(28))) Expect(unsafe.Offsetof(p.ToolParser)).To(Equal(uintptr(32))) Expect(unsafe.Offsetof(p.ReasoningParser)).To(Equal(uintptr(40))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(48))) + Expect(unsafe.Offsetof(p.SpeculativeConfig)).To(Equal(uintptr(48))) + Expect(unsafe.Offsetof(p.EnablePrefixCaching)).To(Equal(uintptr(56))) + Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60))) + Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64))) + Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(80))) }) - It("cSamplingParams matches vllm_sampling_params (ABI v2)", func() { + It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() { var p cSamplingParams Expect(unsafe.Offsetof(p.Temperature)).To(Equal(uintptr(0))) Expect(unsafe.Offsetof(p.TopP)).To(Equal(uintptr(4))) @@ -55,7 +60,9 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.NStructuredChoice)).To(Equal(uintptr(96))) Expect(unsafe.Offsetof(p.StructuredGrammar)).To(Equal(uintptr(104))) Expect(unsafe.Offsetof(p.StructuredJSONObject)).To(Equal(uintptr(112))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120))) + Expect(unsafe.Offsetof(p.LogitsProcessor)).To(Equal(uintptr(120))) + Expect(unsafe.Offsetof(p.LogitsProcessorUserData)).To(Equal(uintptr(128))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136))) }) It("cCompletion matches vllm_completion", func() { @@ -83,6 +90,112 @@ var _ = Describe("parseOptions", func() { }}) Expect(lo).To(Equal(loadOptions{})) }) + + It("carries a speculative_config JSON value through the legacy options list", func() { + // strings.Cut splits on the FIRST colon only, so a JSON object value + // survives the "key:value" spelling intact. + lo := parseOptions(&pb.ModelOptions{Options: []string{ + `speculative_config:{"method":"mtp","num_speculative_tokens":1}`, + }}) + Expect(lo.speculativeConfig).To(Equal(`{"method":"mtp","num_speculative_tokens":1}`)) + }) +}) + +var _ = Describe("engine_args", func() { + It("maps every load knob onto the C model params", func() { + lo := parseOptions(&pb.ModelOptions{EngineArgs: `{ + "block_size": 64, + "num_blocks": 1024, + "max_model_len": 16384, + "max_num_seqs": 32, + "max_num_batched_tokens": 8192, + "enable_prefix_caching": true, + "scheduling_policy": "lpm", + "tool_parser": "qwen3", + "reasoning_parser": "deepseek_r1", + "tokenizer_config": "/models/tok/tokenizer_config.json" + }`}) + Expect(lo.blockSize).To(Equal(int32(64))) + Expect(lo.numBlocks).To(Equal(int32(1024))) + Expect(lo.maxModelLen).To(Equal(int32(16384))) + Expect(lo.maxNumSeqs).To(Equal(int32(32))) + Expect(lo.maxNumBatchedTokens).To(Equal(int32(8192))) + Expect(lo.enablePrefixCaching).To(Equal(int32(1))) + Expect(lo.schedulingPolicy).To(Equal("lpm")) + Expect(lo.toolParser).To(Equal("qwen3")) + Expect(lo.reasoningParser).To(Equal("deepseek_r1")) + Expect(lo.tokenizerConfigPath).To(Equal("/models/tok/tokenizer_config.json")) + }) + + It("re-marshals a nested speculative_config object to JSON for the engine", func() { + lo := parseOptions(&pb.ModelOptions{EngineArgs: `{ + "speculative_config": {"method": "mtp", "num_speculative_tokens": 1} + }`}) + Expect(lo.speculativeConfig).To(MatchJSON(`{"method":"mtp","num_speculative_tokens":1}`)) + }) + + It("re-marshals a nested kv_transfer_config object (LMCache) to JSON", func() { + lo := parseOptions(&pb.ModelOptions{EngineArgs: `{ + "kv_transfer_config": { + "kv_connector": "LMCacheConnector", + "kv_role": "kv_both", + "kv_connector_extra_config": {"host": "127.0.0.1", "port": 65432} + } + }`}) + Expect(lo.kvTransferConfig).To(MatchJSON(`{ + "kv_connector":"LMCacheConnector", + "kv_role":"kv_both", + "kv_connector_extra_config":{"host":"127.0.0.1","port":65432} + }`)) + }) + + It("accepts a pre-encoded JSON string for the object-valued knobs", func() { + // A config written by hand (or round-tripped through a flat store) may + // carry the object as a string; both spellings reach the engine the same. + lo := parseOptions(&pb.ModelOptions{EngineArgs: `{ + "speculative_config": "{\"method\":\"ngram\",\"num_speculative_tokens\":4}" + }`}) + Expect(lo.speculativeConfig).To(MatchJSON(`{"method":"ngram","num_speculative_tokens":4}`)) + }) + + It("maps enable_prefix_caching false onto the force-OFF tri-state", func() { + // The C ABI tri-state is 0=model default, 1=on, 2=off, so an explicit + // `false` must NOT collapse to the 0 that means "let the model decide". + lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_prefix_caching": false}`}) + Expect(lo.enablePrefixCaching).To(Equal(int32(2))) + }) + + It("leaves the prefix-caching tri-state at the model default when unset", func() { + lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`}) + Expect(lo.enablePrefixCaching).To(Equal(int32(0))) + }) + + It("accepts the radix-attention alias upstream documents for prefix caching", func() { + lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_radix_attention": true}`}) + Expect(lo.enablePrefixCaching).To(Equal(int32(1))) + }) + + It("lets engine_args override the legacy options list", func() { + lo := parseOptions(&pb.ModelOptions{ + Options: []string{"max_num_seqs:8", "block_size:16"}, + EngineArgs: `{"max_num_seqs": 64}`, + }) + Expect(lo.maxNumSeqs).To(Equal(int32(64))) // engine_args wins + Expect(lo.blockSize).To(Equal(int32(16))) // untouched keys survive + }) + + It("ignores malformed engine_args rather than failing the load", func() { + lo := parseOptions(&pb.ModelOptions{ + Options: []string{"max_num_seqs:8"}, + EngineArgs: `{not json`, + }) + Expect(lo.maxNumSeqs).To(Equal(int32(8))) + }) + + It("ignores unknown keys", func() { + lo := parseOptions(&pb.ModelOptions{EngineArgs: `{"gpu_memory_utilization": 0.9}`}) + Expect(lo).To(Equal(loadOptions{})) + }) }) var _ = Describe("samplingFromPredict", func() { diff --git a/core/config/vllm_spec.go b/core/config/vllm_spec.go new file mode 100644 index 000000000..059fa9fe1 --- /dev/null +++ b/core/config/vllm_spec.go @@ -0,0 +1,117 @@ +package config + +// Speculative-decoding auto-defaults for the vllm-cpp backend, the safetensors +// counterpart of the GGUF/llama.cpp hook in mtp.go. +// +// The two engines detect and spell the same feature differently. llama.cpp +// reads `.nextn_predict_layers` out of the GGUF header and takes +// `spec_type:draft-mtp` in `options:`; vllm.cpp reads `mtp_num_hidden_layers` +// out of the checkpoint's config.json and takes vLLM's own +// `--speculative-config` JSON, which LocalAI carries in `engine_args`. The +// engine resolves the draft depth and the default k itself, so the config only +// has to name the method. + +import ( + "encoding/json" + + "github.com/mudler/xlog" +) + +// hfSpecConfig is the subset of a HuggingFace config.json that decides whether +// speculative decoding can be auto-enabled. +type hfSpecConfig struct { + ModelType string `json:"model_type"` + // MtpNumHiddenLayers is the MTP head depth (upstream speculative.py reads + // it as n_predict for the qwen3_5 / qwen3_5_moe families). + MtpNumHiddenLayers uint32 `json:"mtp_num_hidden_layers"` + // DFlashConfig marks a z-lab DFlash DRAFT checkpoint (mask_token_id + + // target_layer_ids). Its presence means this repo is a draft, not a + // servable target. + DFlashConfig json.RawMessage `json:"dflash_config"` + // TextConfig is where multimodal checkpoints nest the language-model + // config, and therefore the MTP depth. + TextConfig *hfSpecConfig `json:"text_config"` +} + +// parseHFSpecConfig decodes the speculative-relevant subset of a config.json. +// A document that does not parse yields nothing rather than an error: detection +// is best-effort and must never break an import. +func parseHFSpecConfig(configJSON []byte) (hfSpecConfig, bool) { + if len(configJSON) == 0 { + return hfSpecConfig{}, false + } + var c hfSpecConfig + if err := json.Unmarshal(configJSON, &c); err != nil { + xlog.Debug("[vllm-spec] config.json did not parse; skipping detection", "error", err) + return hfSpecConfig{}, false + } + return c, true +} + +// IsDFlashDraftConfig reports whether a HuggingFace config.json describes a +// DFlash DRAFT checkpoint. Unlike MTP - whose head ships inside the target +// checkpoint's `mtp.*` tensors - a DFlash draft is its own repo that can only +// run paired with a target it verifies against, so it must never be configured +// as a standalone model. +func IsDFlashDraftConfig(configJSON []byte) bool { + c, ok := parseHFSpecConfig(configJSON) + if !ok { + return false + } + return len(c.DFlashConfig) > 0 || + (c.TextConfig != nil && len(c.TextConfig.DFlashConfig) > 0) +} + +// HasSafetensorsMTPHead reports whether a HuggingFace config.json declares a +// self-speculating Multi-Token Prediction head, returning its depth. The depth +// is informational: vllm.cpp resolves n_predict and the default +// num_speculative_tokens from the checkpoint itself. +// +// DFlash drafts are excluded for the same reason `gemma4-assistant` GGUFs are +// excluded from the llama.cpp hook: they carry head metadata but cannot +// self-speculate. +// +// NOTE this is a safetensors-only signal. vllm.cpp rejects an MTP config over a +// GGUF source, because the `mtp.*` draft tensors only exist in the safetensors +// checkpoint - so the GGUF import path must not use this. +func HasSafetensorsMTPHead(configJSON []byte) (uint32, bool) { + c, ok := parseHFSpecConfig(configJSON) + if !ok { + return 0, false + } + if IsDFlashDraftConfig(configJSON) { + return 0, false + } + n := c.MtpNumHiddenLayers + if n == 0 && c.TextConfig != nil { + n = c.TextConfig.MtpNumHiddenLayers + } + return n, n > 0 +} + +// ApplyVLLMSpeculativeDefaults enables MTP speculative decoding in cfg's +// engine_args when nothing is configured there yet. It is a no-op when the user +// already set a speculative_config, so an explicit choice (a different method, +// an explicit k, a DFlash draft) is never clobbered. +// +// `layers` is the detected head depth and is only used for the diagnostic log +// line - the engine derives the real k from the checkpoint. +func ApplyVLLMSpeculativeDefaults(cfg *ModelConfig, layers uint32) { + if cfg == nil { + return + } + if _, set := cfg.EngineArgs["speculative_config"]; set { + xlog.Debug("[vllm-spec] MTP head detected but speculative_config already configured; leaving user choice intact", + "name", cfg.Name, "mtp_num_hidden_layers", layers) + return + } + if cfg.EngineArgs == nil { + cfg.EngineArgs = map[string]any{} + } + // Only the method: vllm.cpp defaults num_speculative_tokens to the + // checkpoint's own n_predict (speculative.py:865-875), which is the right + // value far more reliably than anything guessable here. + cfg.EngineArgs["speculative_config"] = map[string]any{"method": "mtp"} + xlog.Info("[vllm-spec] MTP head detected; enabling mtp speculative decoding", + "name", cfg.Name, "mtp_num_hidden_layers", layers) +} diff --git a/core/config/vllm_spec_test.go b/core/config/vllm_spec_test.go new file mode 100644 index 000000000..fb3854247 --- /dev/null +++ b/core/config/vllm_spec_test.go @@ -0,0 +1,117 @@ +package config_test + +import ( + . "github.com/mudler/LocalAI/core/config" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("vllm-cpp speculative-decoding auto-defaults", func() { + Context("HasSafetensorsMTPHead", func() { + It("detects a top-level mtp_num_hidden_layers", func() { + n, ok := HasSafetensorsMTPHead([]byte(`{ + "model_type": "qwen3_5_moe", + "mtp_num_hidden_layers": 1 + }`)) + Expect(ok).To(BeTrue()) + Expect(n).To(Equal(uint32(1))) + }) + + It("detects the head nested under text_config", func() { + // Multimodal checkpoints nest the language-model config, which is + // where the MTP depth lives (mirrors the engine's own resolution + // off config.raw text_config). + n, ok := HasSafetensorsMTPHead([]byte(`{ + "model_type": "qwen3_5_moe", + "text_config": {"mtp_num_hidden_layers": 2} + }`)) + Expect(ok).To(BeTrue()) + Expect(n).To(Equal(uint32(2))) + }) + + It("reports no head when the key is absent", func() { + n, ok := HasSafetensorsMTPHead([]byte(`{"model_type": "llama"}`)) + Expect(ok).To(BeFalse()) + Expect(n).To(BeZero()) + }) + + It("reports no head for a zero depth", func() { + _, ok := HasSafetensorsMTPHead([]byte(`{"mtp_num_hidden_layers": 0}`)) + Expect(ok).To(BeFalse()) + }) + + It("ignores a DFlash draft checkpoint", func() { + // A DFlash draft is a SEPARATE checkpoint that cannot serve alone: + // it needs a target to verify against. Same exclusion the GGUF path + // makes for gemma4-assistant drafts. + _, ok := HasSafetensorsMTPHead([]byte(`{ + "model_type": "qwen3_dflash", + "mtp_num_hidden_layers": 1, + "dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0, 1]} + }`)) + Expect(ok).To(BeFalse()) + }) + + It("reports no head on unparseable JSON", func() { + _, ok := HasSafetensorsMTPHead([]byte(`{not json`)) + Expect(ok).To(BeFalse()) + }) + + It("reports no head on empty input", func() { + _, ok := HasSafetensorsMTPHead(nil) + Expect(ok).To(BeFalse()) + }) + }) + + Context("IsDFlashDraftConfig", func() { + It("recognises a draft by its dflash_config block", func() { + Expect(IsDFlashDraftConfig([]byte(`{ + "dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0]} + }`))).To(BeTrue()) + }) + + It("does not flag an ordinary checkpoint", func() { + Expect(IsDFlashDraftConfig([]byte(`{"model_type": "qwen3_5_moe"}`))).To(BeFalse()) + }) + }) + + Context("ApplyVLLMSpeculativeDefaults", func() { + It("writes the mtp method into engine_args", func() { + cfg := &ModelConfig{Name: "qwen"} + ApplyVLLMSpeculativeDefaults(cfg, 1) + Expect(cfg.EngineArgs).To(HaveKey("speculative_config")) + spec, ok := cfg.EngineArgs["speculative_config"].(map[string]any) + Expect(ok).To(BeTrue()) + Expect(spec["method"]).To(Equal("mtp")) + }) + + It("leaves an existing speculative_config alone", func() { + cfg := &ModelConfig{ + Name: "qwen", + LLMConfig: LLMConfig{ + EngineArgs: map[string]any{ + "speculative_config": map[string]any{"method": "ngram", "num_speculative_tokens": 4}, + }, + }, + } + ApplyVLLMSpeculativeDefaults(cfg, 1) + spec := cfg.EngineArgs["speculative_config"].(map[string]any) + Expect(spec["method"]).To(Equal("ngram")) + }) + + It("preserves unrelated engine_args keys", func() { + cfg := &ModelConfig{ + Name: "qwen", + LLMConfig: LLMConfig{EngineArgs: map[string]any{"max_num_seqs": 32}}, + } + ApplyVLLMSpeculativeDefaults(cfg, 1) + Expect(cfg.EngineArgs).To(HaveKeyWithValue("max_num_seqs", 32)) + Expect(cfg.EngineArgs).To(HaveKey("speculative_config")) + }) + + It("tolerates a nil config", func() { + Expect(func() { ApplyVLLMSpeculativeDefaults(nil, 1) }).ToNot(Panic()) + }) + }) +}) diff --git a/core/gallery/importers/llama-cpp.go b/core/gallery/importers/llama-cpp.go index 0804ce34f..7ffb5e566 100644 --- a/core/gallery/importers/llama-cpp.go +++ b/core/gallery/importers/llama-cpp.go @@ -298,7 +298,15 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error) // imported configs already carry spec_type:draft-mtp before the model is // ever loaded - users see it in the YAML preview rather than discovering // it after the first start. - maybeApplyMTPDefaults(&modelConfig, details, &cfg) + // + // vllm-cpp is excluded on both counts: `spec_type:*` are llama.cpp option + // keys it does not read, and vllm.cpp rejects an MTP config over a GGUF + // source outright (the `mtp.*` draft tensors exist only in the safetensors + // checkpoint). Its MTP auto-config runs in the vllm importer instead, over + // the safetensors config.json. + if backend != "vllm-cpp" { + maybeApplyMTPDefaults(&modelConfig, details, &cfg) + } data, err := yaml.Marshal(modelConfig) if err != nil { diff --git a/core/gallery/importers/vllm.go b/core/gallery/importers/vllm.go index 8fdbce5a6..279be5d2d 100644 --- a/core/gallery/importers/vllm.go +++ b/core/gallery/importers/vllm.go @@ -1,13 +1,21 @@ package importers import ( + "context" "encoding/json" + "fmt" + "io" + "net/http" "path/filepath" "strings" + "time" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/httpclient" + "github.com/mudler/xlog" "go.yaml.in/yaml/v2" ) @@ -107,6 +115,12 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) { // vllm python backend, so use_tokenizer_template carries over), but // tool/reasoning parsing is the engine's own autoparser pipeline - // the vllm-python tool_parser/reasoning_parser options don't apply. + // + // Auto-detect a Multi-Token Prediction head, the safetensors analogue + // of the llama-cpp importer's GGUF hook, so a freshly imported + // Qwen3.5 / Qwen3.6 config already carries speculative decoding in its + // engine_args instead of leaving the throughput on the table. + maybeApplyVLLMSpeculativeDefaults(&modelConfig, details) } else { // Auto-detect tool_parser and reasoning_parser for known model families. // Surfacing them in the generated YAML lets users see and edit the choices. @@ -132,3 +146,89 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) { ConfigFile: string(data), }, nil } + +// maxSpecConfigProbeBytes caps the config.json body we read. Real ones are a +// few KB; the cap keeps a hostile or mislabelled URL from streaming into the +// importer. +const maxSpecConfigProbeBytes = 1 << 20 // 1 MiB + +// specConfigProbeTimeout bounds the config.json fetch. Detection is an +// optimisation, so it must never hold an import open for long. +const specConfigProbeTimeout = 30 * time.Second + +// specConfigFetcher is the seam the config.json probe goes through, so tests can +// drive the whole import path without a network round trip. +var specConfigFetcher = fetchProbeBody + +// maybeApplyVLLMSpeculativeDefaults fetches the repository's config.json and, +// when it declares a Multi-Token Prediction head, enables MTP speculative +// decoding in the emitted engine_args. This is the safetensors counterpart of +// the llama-cpp importer's GGUF header probe. +// +// Every failure is non-fatal and logged at debug: a network blip, a private +// repo, or a config.json this doesn't understand must leave the import working +// exactly as it did before, just without the speculative default. +func maybeApplyVLLMSpeculativeDefaults(modelConfig *config.ModelConfig, details Details) { + probeURL := vllmSpecProbeURL(details) + if probeURL == "" { + return + } + + body, err := specConfigFetcher(probeURL) + if err != nil { + xlog.Debug("[vllm-spec-importer] could not read config.json for MTP detection", "uri", probeURL, "error", err) + return + } + + applySpecFromConfigJSON(modelConfig, body, details.URI) +} + +// applySpecFromConfigJSON is the decision half of the probe, split out so it can +// be exercised without a network round trip. +func applySpecFromConfigJSON(modelConfig *config.ModelConfig, body []byte, uri string) { + if config.IsDFlashDraftConfig(body) { + // A DFlash draft cannot serve on its own - it only proposes tokens for + // a target model to verify. Say so rather than emitting a config that + // would fail at load. + xlog.Warn("[vllm-spec-importer] this repository is a DFlash DRAFT checkpoint, not a servable model; "+ + "import the TARGET model and point engine_args.speculative_config at this repo "+ + `({"method":"dflash","model":""})`, "uri", uri) + return + } + + n, ok := config.HasSafetensorsMTPHead(body) + if !ok { + return + } + config.ApplyVLLMSpeculativeDefaults(modelConfig, n) +} + +// vllmSpecProbeURL returns the HTTP(S) URL of the repository's config.json, or +// "" when the import isn't backed by a HuggingFace repo we can fetch from (a +// local directory import, an OCI artifact, ...). +func vllmSpecProbeURL(details Details) string { + if details.HuggingFace == nil || details.HuggingFace.ModelID == "" { + return "" + } + return resolveHTTPProbe(downloader.HuggingFacePrefix + details.HuggingFace.ModelID + "/config.json") +} + +// fetchProbeBody GETs a small remote JSON document under a short timeout. +func fetchProbeBody(url string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), specConfigProbeTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := httpclient.NewWithTimeout(specConfigProbeTimeout).Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status %d", resp.StatusCode) + } + return io.ReadAll(io.LimitReader(resp.Body, maxSpecConfigProbeBytes)) +} diff --git a/core/gallery/importers/vllm_spec_internal_test.go b/core/gallery/importers/vllm_spec_internal_test.go new file mode 100644 index 000000000..72b31cbb2 --- /dev/null +++ b/core/gallery/importers/vllm_spec_internal_test.go @@ -0,0 +1,118 @@ +package importers + +import ( + "encoding/json" + "errors" + + "github.com/mudler/LocalAI/core/config" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("vllm-cpp speculative auto-config (importer)", func() { + Context("applySpecFromConfigJSON", func() { + It("enables mtp when the checkpoint declares an MTP head", func() { + cfg := &config.ModelConfig{Name: "qwen3.5"} + applySpecFromConfigJSON(cfg, []byte(`{ + "model_type": "qwen3_5_moe", + "mtp_num_hidden_layers": 1 + }`), "huggingface://Qwen/Qwen3.5-A3B") + Expect(cfg.EngineArgs).To(HaveKeyWithValue("speculative_config", + map[string]any{"method": "mtp"})) + }) + + It("leaves a plain checkpoint untouched", func() { + cfg := &config.ModelConfig{Name: "llama"} + applySpecFromConfigJSON(cfg, []byte(`{"model_type": "llama"}`), "huggingface://meta/llama") + Expect(cfg.EngineArgs).To(BeEmpty()) + }) + + It("refuses to configure a DFlash draft as a servable model", func() { + // The draft only proposes tokens; configuring it standalone would + // produce a model that cannot load. + cfg := &config.ModelConfig{Name: "dflash-draft"} + applySpecFromConfigJSON(cfg, []byte(`{ + "model_type": "qwen3_dflash", + "dflash_config": {"mask_token_id": 151666, "target_layer_ids": [0, 1]} + }`), "huggingface://z-lab/Qwen3.6-27B-DFlash") + Expect(cfg.EngineArgs).To(BeEmpty()) + }) + + It("survives a config.json it cannot parse", func() { + cfg := &config.ModelConfig{Name: "weird"} + Expect(func() { + applySpecFromConfigJSON(cfg, []byte(`404`), "huggingface://a/b") + }).ToNot(Panic()) + Expect(cfg.EngineArgs).To(BeEmpty()) + }) + }) + + Context("Import over a repository with an MTP head", func() { + var restore func() + + BeforeEach(func() { + original := specConfigFetcher + restore = func() { specConfigFetcher = original } + }) + AfterEach(func() { restore() }) + + importWith := func(backend, configJSON string) string { + specConfigFetcher = func(string) ([]byte, error) { + return []byte(configJSON), nil + } + importer := &VLLMImporter{} + out, err := importer.Import(Details{ + URI: "huggingface://Qwen/Qwen3.5-A3B", + Preferences: json.RawMessage(`{"backend": "` + backend + `"}`), + HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"}, + }) + Expect(err).ToNot(HaveOccurred()) + return out.ConfigFile + } + + It("emits engine_args.speculative_config for vllm-cpp", func() { + yaml := importWith("vllm-cpp", `{"model_type":"qwen3_5_moe","mtp_num_hidden_layers":1}`) + Expect(yaml).To(ContainSubstring("engine_args:")) + Expect(yaml).To(ContainSubstring("speculative_config:")) + Expect(yaml).To(ContainSubstring("method: mtp")) + }) + + It("emits nothing speculative for the python vllm backend", func() { + // The python backend has its own speculative surface and its own + // version-dependent MTP support; this hook is vllm-cpp only. + yaml := importWith("vllm", `{"model_type":"qwen3_5_moe","mtp_num_hidden_layers":1}`) + Expect(yaml).NotTo(ContainSubstring("speculative_config")) + }) + + It("emits nothing speculative when the probe fails", func() { + specConfigFetcher = func(string) ([]byte, error) { + return nil, errors.New("network down") + } + importer := &VLLMImporter{} + out, err := importer.Import(Details{ + URI: "huggingface://Qwen/Qwen3.5-A3B", + Preferences: json.RawMessage(`{"backend": "vllm-cpp"}`), + HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(out.ConfigFile).NotTo(ContainSubstring("speculative_config")) + }) + }) + + Context("vllmSpecProbeURL", func() { + It("resolves the repository's config.json to an HTTPS URL", func() { + url := vllmSpecProbeURL(Details{ + URI: "huggingface://Qwen/Qwen3.5-A3B", + HuggingFace: &hfapi.ModelDetails{ModelID: "Qwen/Qwen3.5-A3B"}, + }) + Expect(url).To(ContainSubstring("Qwen/Qwen3.5-A3B")) + Expect(url).To(HaveSuffix("config.json")) + Expect(url).To(HavePrefix("https://")) + }) + + It("skips the probe when there is no HuggingFace repo behind the import", func() { + Expect(vllmSpecProbeURL(Details{URI: "/models/local-dir"})).To(BeEmpty()) + }) + }) +}) diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index f7cc155a8..16060420e 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -889,6 +889,166 @@ options: The full list of registered parsers lives in `sglang.srt.function_call` and `sglang.srt.parser.reasoning_parser`. +### vllm.cpp + +[vllm.cpp](https://github.com/mudler/vllm.cpp) is the LocalAI team's C++ port of +vLLM: the same continuous-batching scheduler, paged KV cache and prefix caching, +with no Python at inference time. It consumes either a HuggingFace safetensors +model directory or a `.gguf` file, and applies the model's chat template, +tool-call parsing and reasoning split engine-side. + +#### Setup + +```yaml +name: vllm-cpp +backend: vllm-cpp +parameters: + model: "Qwen/Qwen3-4B" +context_size: 8192 +template: + use_tokenizer_template: true +``` + +#### Configuring the engine with `engine_args` + +The same `engine_args:` map the vLLM and SGLang backends accept is honoured +here, with keys spelled exactly as vLLM's own CLI flags - so a `speculative_config` +or `kv_transfer_config` block written for vLLM works verbatim. Unknown keys are +ignored rather than fatal; the engine validates the documents it is handed and +reports a precise error at load. + +```yaml +name: qwen35-a3b +backend: vllm-cpp +parameters: + model: "Qwen/Qwen3.5-A3B" +context_size: 16384 +template: + use_tokenizer_template: true +engine_args: + # KV cache sizing: num_blocks * block_size tokens of cache. + block_size: 32 + num_blocks: 1024 + # Concurrency and the per-step chunked-prefill token budget. + max_num_seqs: 32 + max_num_batched_tokens: 8192 + # Automatic prefix caching. Omit to keep the model's own default + # (on for dense models, off for hybrid / attention-free ones). + enable_prefix_caching: true + # Scheduler admission order: fcfs (default), priority, or lpm + # (cache-aware longest-prefix-match; needs prefix caching to have any effect). + scheduling_policy: lpm +``` + +| Key | Meaning | Default | +|-----|---------|---------| +| `block_size` | KV-cache block size, in tokens per block | 32 | +| `num_blocks` | KV-cache blocks to allocate | 256 | +| `max_model_len` | Max sequence length; also settable as `context_size` / `max_model_len` | model config | +| `max_num_seqs` | Max concurrent sequences the scheduler admits | 8 | +| `max_num_batched_tokens` | Per-step chunked-prefill token budget | per-arch (2048 dense, 4096/8192 MoE) | +| `enable_prefix_caching` | Automatic prefix caching; `enable_radix_attention` is an accepted alias | model default | +| `scheduling_policy` | `fcfs`, `priority`, or `lpm` | `fcfs` | +| `tool_parser` / `reasoning_parser` | Force a parser instead of chat-template auto-detection | auto | +| `tokenizer_config` | Override the `tokenizer_config.json` the chat template is read from | `/tokenizer_config.json` | +| `speculative_config` | Speculative decoding (see below) | disabled | +| `kv_transfer_config` | External KV connector / LMCache (see below) | none | + +Raising `max_num_batched_tokens` lets more prefill land in a single step, at the +cost of decode latency for requests queued behind it. The default deliberately +does not scale with `max_num_seqs`, which is what keeps a large concurrent +prefill from blowing up the per-step activation on the hybrid architectures. + +#### Speculative decoding + +`speculative_config:` takes the same JSON object as vLLM's +`--speculative-config`. Three methods are supported: + +**MTP** (Multi-Token Prediction) uses a draft head shipped inside the target +checkpoint's own `mtp.*` tensors, so there is no second model to download. It +requires a **safetensors** checkpoint - the `mtp.*` tensors do not survive GGUF +conversion, and an MTP config over a `.gguf` model is rejected at load. + +```yaml +engine_args: + speculative_config: + method: mtp + # Optional; defaults to the checkpoint's own head depth, which is + # usually the right value. Must be a multiple of that depth. + num_speculative_tokens: 1 +``` + +**DFlash** uses a separate block-diffusion drafter that proposes a whole block +of tokens in one non-autoregressive forward pass. Unlike MTP, the draft is its +own checkpoint, so `model:` is **required**: + +```yaml +engine_args: + speculative_config: + method: dflash + model: z-lab/Qwen3.6-27B-DFlash + num_speculative_tokens: 4 +``` + +**N-gram** needs no draft model at all - it proposes from the prompt's own +suffix history. `num_speculative_tokens` is required: + +```yaml +engine_args: + speculative_config: + method: ngram + num_speculative_tokens: 4 + prompt_lookup_min: 5 + prompt_lookup_max: 5 +``` + +> **Auto-configuration on import.** When you import a safetensors repository +> with `backend: vllm-cpp`, LocalAI reads the checkpoint's `config.json` and, if +> it declares an MTP head (`mtp_num_hidden_layers`), writes +> `speculative_config: {method: mtp}` into the generated `engine_args` for you. +> An explicit `speculative_config` in your own config is never overwritten. +> Importing a DFlash *draft* repository is refused with a warning: a drafter +> cannot serve on its own, so import the target model and point +> `speculative_config.model` at the draft. + +#### External KV cache with LMCache + +`kv_transfer_config:` takes vLLM's `--kv-transfer-config` JSON and selects an +external KV-cache connector. The `lm://` LMCache client lets prefill KV be +stored to and reloaded from a shared `lmcache.v1.server`, so a prefix computed +by one replica does not have to be recomputed by the next: + +```yaml +engine_args: + kv_transfer_config: + kv_connector: LMCacheConnector + kv_role: kv_both # required whenever kv_connector is set + kv_connector_extra_config: + host: 127.0.0.1 + port: 65432 +``` + +`kv_role` is one of `kv_producer` (store only), `kv_consumer` (load only), or +`kv_both`. An unregistered connector name, a missing role, or a malformed +document fails the load with an explicit error rather than silently running +without the cache. + +#### Legacy `options:` list + +Earlier versions configured this backend through the flat `options:` list, and +those configs keep working. Every key in the table above is still read from +there in `key:value` form, and `engine_args` wins on any key set in both: + +```yaml +options: + - max_num_seqs:32 + - enable_prefix_caching:true +``` + +New configs should prefer `engine_args:`, which is the only place the nested +`speculative_config` / `kv_transfer_config` documents can be written naturally +rather than as a single-line JSON string. + ### Transformers [Transformers](https://huggingface.co/docs/transformers/index) is a State-of-the-art Machine Learning library for PyTorch, TensorFlow, and JAX.