From 13b780d38759d225992fa36d42593a844276a5aa Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 15:24:35 +0200 Subject: [PATCH] chore: :arrow_up: Update mudler/vllm.cpp to `60990ee784101f74f6d1775575e9e89dfb26f73a` (#12014) * :arrow_up: Update mudler/vllm.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(vllm-cpp): mirror ABI v26 model params Mirror the new KV-cache dtype and sliding-window fields so the Go POD matches the bumped vllm.cpp header on LP64. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/go/vllm-cpp/Makefile | 2 +- backend/go/vllm-cpp/govllmcpp.go | 47 +++++++++++++++-------------- backend/go/vllm-cpp/vllmcpp_test.go | 22 ++++++++------ 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 598d814c3..0e379f090 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=6bf3abb580982f4fd2e4525ef37802ee0ce28981 +VLLM_CPP_VERSION?=60990ee784101f74f6d1775575e9e89dfb26f73a # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index 20e587a6b..db7555cd2 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v23). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v26). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 23 +const abiVersion = 26 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -51,28 +51,28 @@ const ( vllmOK = 0 ) -// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the -// interior needs no padding on LP64, but the struct is 8-aligned (it holds -// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets -// and total size are asserted in vllmcpp_test.go. +// cModelParams mirrors vllm_model_params. Go's natural alignment and the +// explicit pad after LanguageModelOnly match the C layout on LP64. Offsets and +// total size are asserted in vllmcpp_test.go. type cModelParams struct { - ModelPath uintptr // const char* - TokenizerConfigPath uintptr // const char*; NULL = /... (ABI v9) - BlockSize int32 - NumBlocks int32 - MaxModelLen int32 - MaxNumSeqs int32 - ToolParser uintptr // const char*; NULL = auto-detect (ABI v4) - ReasoningParser uintptr // const char*; NULL = auto-detect (ABI v5) - SpeculativeConfig uintptr // const char* JSON; NULL = no speculation (ABI v6) - EnablePrefixCaching int32 // tri-state 0/1/2 (ABI v7) - MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9) - SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) - KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) - OffloadConfig uintptr // const char* JSON; NULL = no weight offload - EnableJumpForward int32 // tri-state 0/1/2 (ABI v10) - // v14/v16 tail. LocalAI sets none of these (0 is "auto" for the device and - // "unset" for both sizing knobs, i.e. the pre-v14 engine byte for byte), but + ModelPath uintptr // const char* + TokenizerConfigPath uintptr // const char*; NULL = /... (ABI v9) + BlockSize int32 + NumBlocks int32 + MaxModelLen int32 + MaxNumSeqs int32 + ToolParser uintptr // const char*; NULL = auto-detect (ABI v4) + ReasoningParser uintptr // const char*; NULL = auto-detect (ABI v5) + SpeculativeConfig uintptr // const char* JSON; NULL = no speculation (ABI v6) + EnablePrefixCaching int32 // tri-state 0/1/2 (ABI v7) + MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9) + SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) + KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) + OffloadConfig uintptr // const char* JSON; NULL = no weight offload + EnableJumpForward int32 // tri-state 0/1/2 (ABI v10) + DisableSlidingWindow int32 // tri-state 0/1/2 (ABI v26) + // LocalAI sets none of the device and sizing fields below (0 is "auto" for + // the device and "unset" for both sizing knobs), but // the fields MUST be mirrored: the C side reads sizeof(vllm_model_params) // bytes off the pointer we hand it, so a Go struct that stopped at // EnableJumpForward would have vllm_engine_load read 24 bytes past our @@ -84,6 +84,7 @@ type cModelParams struct { _ [4]byte LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19) MMProjPath uintptr // const char*; NULL = no GGUF projector (ABI v22) + KVCacheDType uintptr // const char*; NULL = auto (ABI v24) } // cSamplingParams mirrors vllm_sampling_params (structured fields included). diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 55f6e211a..201fd1692 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v23) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v26) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(23)) + Expect(abiVersion).To(Equal(26)) }) It("cModelParams matches vllm_model_params", func() { @@ -44,15 +44,17 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72))) Expect(unsafe.Offsetof(p.OffloadConfig)).To(Equal(uintptr(80))) Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(88))) - Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(92))) - // 96: gpu_memory_utilization is a double, so it takes the next + Expect(unsafe.Offsetof(p.DisableSlidingWindow)).To(Equal(uintptr(92))) + Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(96))) + // 104: gpu_memory_utilization is a double, so it takes the next // 8-aligned slot after the int32 pair. Go pads identically. - Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(96))) - Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104))) - Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112))) - Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120))) - Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(128))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136))) + Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(112))) + Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(120))) + Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(128))) + Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(136))) + Expect(unsafe.Offsetof(p.KVCacheDType)).To(Equal(uintptr(144))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(152))) }) It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() {