mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-14 15:18:03 -04:00
chore: ⬆️ Update mudler/vllm.cpp to 60990ee784101f74f6d1775575e9e89dfb26f73a (#12014)
* ⬆️ Update mudler/vllm.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(vllm-cpp): mirror ABI v26 model params Mirror the new KV-cache dtype and sliding-window fields so the Go POD matches the bumped vllm.cpp header on LP64. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
3 files changed
+37
-34
No files matched your search
@@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
|
||||
|
||||
# vllm.cpp version
|
||||
VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
|
||||
VLLM_CPP_VERSION?=6bf3abb580982f4fd2e4525ef37802ee0ce28981
|
||||
VLLM_CPP_VERSION?=60990ee784101f74f6d1775575e9e89dfb26f73a
|
||||
|
||||
# MLX GEMM provider (darwin/metal only; see the metal branch below for why).
|
||||
# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package main
|
||||
|
||||
// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v23).
|
||||
// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v26).
|
||||
//
|
||||
// The structs below are hand-mirrored PODs of the C declarations, with
|
||||
// explicit padding so the Go layout matches the C layout on linux/darwin
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
// the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks
|
||||
// the two against each other, because a mismatch is only caught at runtime by
|
||||
// registerLib, where it takes the backend down on every load (issue #11379).
|
||||
const abiVersion = 23
|
||||
const abiVersion = 26
|
||||
|
||||
// The ABI's tri-state toggles (enable_prefix_caching ABI v7,
|
||||
// enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
|
||||
@@ -51,28 +51,28 @@ const (
|
||||
vllmOK = 0
|
||||
)
|
||||
|
||||
// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the
|
||||
// interior needs no padding on LP64, but the struct is 8-aligned (it holds
|
||||
// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets
|
||||
// and total size are asserted in vllmcpp_test.go.
|
||||
// cModelParams mirrors vllm_model_params. Go's natural alignment and the
|
||||
// explicit pad after LanguageModelOnly match the C layout on LP64. Offsets and
|
||||
// total size are asserted in vllmcpp_test.go.
|
||||
type cModelParams struct {
|
||||
ModelPath uintptr // const char*
|
||||
TokenizerConfigPath uintptr // const char*; NULL = <model_dir>/... (ABI v9)
|
||||
BlockSize int32
|
||||
NumBlocks int32
|
||||
MaxModelLen int32
|
||||
MaxNumSeqs int32
|
||||
ToolParser uintptr // const char*; NULL = auto-detect (ABI v4)
|
||||
ReasoningParser uintptr // const char*; NULL = auto-detect (ABI v5)
|
||||
SpeculativeConfig uintptr // const char* JSON; NULL = no speculation (ABI v6)
|
||||
EnablePrefixCaching int32 // tri-state 0/1/2 (ABI v7)
|
||||
MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9)
|
||||
SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9)
|
||||
KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9)
|
||||
OffloadConfig uintptr // const char* JSON; NULL = no weight offload
|
||||
EnableJumpForward int32 // tri-state 0/1/2 (ABI v10)
|
||||
// v14/v16 tail. LocalAI sets none of these (0 is "auto" for the device and
|
||||
// "unset" for both sizing knobs, i.e. the pre-v14 engine byte for byte), but
|
||||
ModelPath uintptr // const char*
|
||||
TokenizerConfigPath uintptr // const char*; NULL = <model_dir>/... (ABI v9)
|
||||
BlockSize int32
|
||||
NumBlocks int32
|
||||
MaxModelLen int32
|
||||
MaxNumSeqs int32
|
||||
ToolParser uintptr // const char*; NULL = auto-detect (ABI v4)
|
||||
ReasoningParser uintptr // const char*; NULL = auto-detect (ABI v5)
|
||||
SpeculativeConfig uintptr // const char* JSON; NULL = no speculation (ABI v6)
|
||||
EnablePrefixCaching int32 // tri-state 0/1/2 (ABI v7)
|
||||
MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9)
|
||||
SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9)
|
||||
KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9)
|
||||
OffloadConfig uintptr // const char* JSON; NULL = no weight offload
|
||||
EnableJumpForward int32 // tri-state 0/1/2 (ABI v10)
|
||||
DisableSlidingWindow int32 // tri-state 0/1/2 (ABI v26)
|
||||
// LocalAI sets none of the device and sizing fields below (0 is "auto" for
|
||||
// the device and "unset" for both sizing knobs), but
|
||||
// the fields MUST be mirrored: the C side reads sizeof(vllm_model_params)
|
||||
// bytes off the pointer we hand it, so a Go struct that stopped at
|
||||
// EnableJumpForward would have vllm_engine_load read 24 bytes past our
|
||||
@@ -84,6 +84,7 @@ type cModelParams struct {
|
||||
_ [4]byte
|
||||
LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19)
|
||||
MMProjPath uintptr // const char*; NULL = no GGUF projector (ABI v22)
|
||||
KVCacheDType uintptr // const char*; NULL = auto (ABI v24)
|
||||
}
|
||||
|
||||
// cSamplingParams mirrors vllm_sampling_params (structured fields included).
|
||||
|
||||
@@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) {
|
||||
RunSpecs(t, "vllm-cpp suite")
|
||||
}
|
||||
|
||||
// The Go POD mirrors must match the C struct layout of vllm.h (ABI v23)
|
||||
// The Go POD mirrors must match the C struct layout of vllm.h (ABI v26)
|
||||
// byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
|
||||
// amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
|
||||
var _ = Describe("C ABI struct mirrors", func() {
|
||||
@@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() {
|
||||
// VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile).
|
||||
// Moving the pin past this without growing the mirrors below ships a
|
||||
// backend that refuses every load at startup (issue #11379).
|
||||
Expect(abiVersion).To(Equal(23))
|
||||
Expect(abiVersion).To(Equal(26))
|
||||
})
|
||||
|
||||
It("cModelParams matches vllm_model_params", func() {
|
||||
@@ -44,15 +44,17 @@ var _ = Describe("C ABI struct mirrors", func() {
|
||||
Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72)))
|
||||
Expect(unsafe.Offsetof(p.OffloadConfig)).To(Equal(uintptr(80)))
|
||||
Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(88)))
|
||||
Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(92)))
|
||||
// 96: gpu_memory_utilization is a double, so it takes the next
|
||||
Expect(unsafe.Offsetof(p.DisableSlidingWindow)).To(Equal(uintptr(92)))
|
||||
Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(96)))
|
||||
// 104: gpu_memory_utilization is a double, so it takes the next
|
||||
// 8-aligned slot after the int32 pair. Go pads identically.
|
||||
Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(96)))
|
||||
Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104)))
|
||||
Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112)))
|
||||
Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120)))
|
||||
Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(128)))
|
||||
Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136)))
|
||||
Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(104)))
|
||||
Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(112)))
|
||||
Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(120)))
|
||||
Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(128)))
|
||||
Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(136)))
|
||||
Expect(unsafe.Offsetof(p.KVCacheDType)).To(Equal(uintptr(144)))
|
||||
Expect(unsafe.Sizeof(p)).To(Equal(uintptr(152)))
|
||||
})
|
||||
|
||||
It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() {
|
||||
|
||||
Reference in new issue
Block a user