diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 3acc42577..75df73ed5 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=eec09bed5a03457837b499781c23d8e44f106813 +VLLM_CPP_VERSION?=f384edcd25950bdf785a17d9230f249b99bc841b # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the # server, examples and tests of the engine are never built here. diff --git a/backend/go/vllm-cpp/backend.go b/backend/go/vllm-cpp/backend.go index a53bf951d..d82cc320a 100644 --- a/backend/go/vllm-cpp/backend.go +++ b/backend/go/vllm-cpp/backend.go @@ -146,6 +146,7 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error { mp.MaxNumBatchedTokens = v.opts.maxNumBatchedTokens } mp.EnablePrefixCaching = v.opts.enablePrefixCaching + mp.EnableJumpForward = v.opts.enableJumpForward // Every string below is borrowed by C for the duration of the load call // only (the library copies what it keeps), so the backing slices just have @@ -172,7 +173,8 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error { "blockSize", mp.BlockSize, "numBlocks", mp.NumBlocks, "maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs, "maxNumBatchedTokens", mp.MaxNumBatchedTokens, - "prefixCaching", prefixCachingName(mp.EnablePrefixCaching), + "prefixCaching", triStateName(mp.EnablePrefixCaching), + "jumpForward", triStateName(mp.EnableJumpForward), "schedulingPolicy", v.opts.schedulingPolicy, "speculativeConfig", v.opts.speculativeConfig, "kvTransferConfig", v.opts.kvTransferConfig) diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index c601f139e..4509489e3 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v9). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -18,24 +18,25 @@ import ( ) // abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h). -const abiVersion = 9 +const abiVersion = 10 -// Prefix-caching tri-state (vllm_model_params.enable_prefix_caching, ABI v7). -// 0 is the model-capability default, which is what a config that says nothing -// about prefix caching must produce. +// The ABI's tri-state toggles (enable_prefix_caching ABI v7, +// enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is +// "defer" - to the model capability for prefix caching, to the environment for +// jump forward. Only 2 is an explicit off. const ( - prefixCachingModelDefault int32 = 0 - prefixCachingOn int32 = 1 - prefixCachingOff int32 = 2 + triStateDefer int32 = 0 + triStateOn int32 = 1 + triStateOff int32 = 2 ) -// prefixCachingName renders the tri-state for the load log line, where "0" -// would otherwise read as "off" rather than "whatever the model defaults to". -func prefixCachingName(state int32) string { +// triStateName renders a tri-state for the load log line, where "0" would +// otherwise read as "off" rather than "whatever the default resolves to". +func triStateName(state int32) string { switch state { - case prefixCachingOn: + case triStateOn: return "on" - case prefixCachingOff: + case triStateOff: return "off" default: return "model-default" @@ -47,9 +48,10 @@ const ( vllmOK = 0 ) -// cModelParams mirrors vllm_model_params. Every field is naturally aligned on -// LP64 (the int32 pairs sit together), so the Go layout needs no explicit -// padding; the offsets are asserted in vllmcpp_test.go. +// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the +// interior needs no padding on LP64, but the struct is 8-aligned (it holds +// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets +// and total size are asserted in vllmcpp_test.go. type cModelParams struct { ModelPath uintptr // const char* TokenizerConfigPath uintptr // const char*; NULL = /... (ABI v9) @@ -64,6 +66,8 @@ type cModelParams struct { MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9) SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) + EnableJumpForward int32 // tri-state 0/1/2 (ABI v10) + _ [4]byte // trailing pad to the struct's 8-byte alignment } // cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields diff --git a/backend/go/vllm-cpp/options.go b/backend/go/vllm-cpp/options.go index b2d9a378e..0cdf4dd4c 100644 --- a/backend/go/vllm-cpp/options.go +++ b/backend/go/vllm-cpp/options.go @@ -42,6 +42,10 @@ type loadOptions struct { // Automatic prefix caching tri-state (ABI v7): 0 = the model-capability // default, 1 = force on, 2 = force off. enablePrefixCaching int32 + // Jump-forward decoding tri-state (ABI v10), SGLang's grammar-speed subset: + // 0 = defer to the environment (VT_ENABLE_JUMP_FORWARD, default off), + // 1 = force on, 2 = force off. + enableJumpForward int32 // Scheduler admission policy (ABI v9): "" = fcfs, else fcfs|priority|lpm. schedulingPolicy string // Engine-side parser selection (ABI v4/v5). Empty = the engine @@ -100,7 +104,11 @@ func applyOptionsList(lo *loadOptions, options []string) { lo.tokenizerConfigPath = strings.TrimSpace(v) case "enable_prefix_caching", "enable_radix_attention": if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil { - lo.enablePrefixCaching = prefixCachingTriState(b) + lo.enablePrefixCaching = boolTriState(b) + } + case "enable_jump_forward": + if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil { + lo.enableJumpForward = boolTriState(b) } } } @@ -145,7 +153,11 @@ func applyEngineArgs(lo *loadOptions, engineArgs string) { lo.kvTransferConfig = jsonDocument(v, lo.kvTransferConfig, k) case "enable_prefix_caching", "enable_radix_attention": if b, ok := v.(bool); ok { - lo.enablePrefixCaching = prefixCachingTriState(b) + lo.enablePrefixCaching = boolTriState(b) + } + case "enable_jump_forward": + if b, ok := v.(bool); ok { + lo.enableJumpForward = boolTriState(b) } default: xlog.Debug("[vllm-cpp] ignoring unknown engine_args key", "key", k) @@ -153,15 +165,16 @@ func applyEngineArgs(lo *loadOptions, engineArgs string) { } } -// prefixCachingTriState maps a YAML/JSON boolean onto the C ABI's tri-state. An +// boolTriState maps a YAML/JSON boolean onto the ABI's tri-state encoding. An // explicit `false` must reach the engine as force-OFF (2), NOT as the 0 that -// means "let the model capability decide" - those differ for the hybrid archs, -// which default the cache off, and for the dense ones, which default it on. -func prefixCachingTriState(on bool) int32 { +// means "defer". The difference is real in both directions: prefix caching +// defaults ON for dense archs and OFF for hybrid ones, and jump forward defers +// to VT_ENABLE_JUMP_FORWARD. +func boolTriState(on bool) int32 { if on { - return prefixCachingOn + return triStateOn } - return prefixCachingOff + return triStateOff } // jsonDocument normalises an object-valued engine_args entry to a JSON string diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 126cff2c3..370bc23d0 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -35,7 +35,10 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60))) Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64))) Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(80))) + Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80))) + // 88, not 84: the struct is 8-aligned (it holds pointers), so the + // trailing int32 is padded out. Go pads identically. + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88))) }) It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() { @@ -192,6 +195,23 @@ var _ = Describe("engine_args", func() { Expect(lo.enablePrefixCaching).To(Equal(int32(1))) }) + It("maps enable_jump_forward onto its own tri-state", func() { + // ABI v10. Same tri-state shape as prefix caching, and the same trap: + // an explicit false must be force-OFF (2), not the 0 that defers to the + // environment. + on := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": true}`}) + Expect(on.enableJumpForward).To(Equal(int32(1))) + off := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": false}`}) + Expect(off.enableJumpForward).To(Equal(int32(2))) + unset := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`}) + Expect(unset.enableJumpForward).To(Equal(int32(0))) + }) + + It("reads enable_jump_forward from the legacy options list too", func() { + lo := parseOptions(&pb.ModelOptions{Options: []string{"enable_jump_forward:true"}}) + Expect(lo.enableJumpForward).To(Equal(int32(1))) + }) + It("lets engine_args override the legacy options list", func() { lo := parseOptions(&pb.ModelOptions{ Options: []string{"max_num_seqs:8", "block_size:16"}, diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index 8c44a1e4c..60ba3d81a 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -948,6 +948,7 @@ engine_args: | `max_num_seqs` | Max concurrent sequences the scheduler admits | 8 | | `max_num_batched_tokens` | Per-step chunked-prefill token budget | per-arch (2048 dense, 4096/8192 MoE) | | `enable_prefix_caching` | Automatic prefix caching; `enable_radix_attention` is an accepted alias | model default | +| `enable_jump_forward` | Jump-forward decoding, which emits grammar-forced tokens without a model step. Only affects constrained requests (`grammar`, JSON schema) | off | | `scheduling_policy` | `fcfs`, `priority`, or `lpm` | `fcfs` | | `tool_parser` / `reasoning_parser` | Force a parser instead of chat-template auto-detection | auto | | `tokenizer_config` | Override the `tokenizer_config.json` the chat template is read from | `/tokenizer_config.json` | @@ -959,6 +960,12 @@ cost of decode latency for requests queued behind it. The default deliberately does not scale with `max_num_seqs`, which is what keeps a large concurrent prefill from blowing up the per-step activation on the hybrid architectures. +`enable_prefix_caching` and `enable_jump_forward` are tri-state at the engine +boundary: omitting the key defers to a default (the model's own capability for +prefix caching, an environment variable for jump forward), while an explicit +`false` forces the feature off. Those are genuinely different - prefix caching +defaults *on* for dense models - so write the key only when you mean to override. + #### Speculative decoding `speculative_config:` takes the same JSON object as vLLM's