mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
df5e2a6d27
vllm.cpp landed ABI v10 on main while this branch was open. The backend's runtime handshake refuses any library whose reported ABI differs from the mirrors', so the pin and the Go PODs move together or not at all. v10 appends one int32, `enable_jump_forward`, AFTER the v9 fields. Nothing else in the config surface changed: the SGLang reconciliation that carried it explicitly dropped its own duplicate scheduler_policy int in favour of the v9 `scheduling_policy` string this branch already wires, and a diff of EngineParams and the server flags across the window turns up jump forward and nothing else. So the exposure is one new knob, `engine_args.enable_jump_forward` - SGLang's grammar-speed subset, which emits grammar-forced tokens without spending a model step and therefore only affects constrained requests. It is the SECOND tri-state on this struct, and it repeats the trap the first one had: 0 is not "off", it is "defer" (to the environment here, to the model capability for prefix caching), so an explicit `false` has to reach the engine as 2. The bool->tri-state helper and the log renderer are now shared rather than duplicated per field, and named for the encoding instead of for prefix caching, since the next tri-state will want them too. The docs say this outright, because "omitting the key" and "setting it false" being different is not guessable. The Go mirror grows the field plus an EXPLICIT trailing pad: the struct is 8-aligned and now ends on a lone int32, so it is 88 bytes rather than 84. The offset assertions cover it, and the real-library handshake spec (VLLM_CPP_LIBRARY against a CPU libvllm.so built at the new pin) confirms the version agrees: 43 specs pass, ABI reported 10. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude Code:claude-opus-5 [ClaudeCode]
103 lines
3.5 KiB
Makefile
103 lines
3.5 KiB
Makefile
CMAKE_ARGS?=
|
|
BUILD_TYPE?=
|
|
NATIVE?=false
|
|
|
|
GOCMD?=go
|
|
GO_TAGS?=
|
|
# nproc doesn't exist on the macOS runners: an empty JOBS turns `-j$(JOBS)`
|
|
# into bare `-j` (unlimited clang jobs), which swap-thrashes the 3-core Mac
|
|
# until the 6h GHA timeout. Fall back to sysctl there, then to a constant.
|
|
JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
|
|
|
# vllm.cpp version
|
|
VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
|
|
VLLM_CPP_VERSION?=f384edcd25950bdf785a17d9230f249b99bc841b
|
|
|
|
# The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
|
|
# server, examples and tests of the engine are never built here.
|
|
CMAKE_ARGS+=-DVLLM_CPP_SERVER=OFF -DVLLM_CPP_BUILD_TESTS=OFF -DVLLM_CPP_BUILD_EXAMPLES=OFF
|
|
CMAKE_ARGS+=-DCMAKE_BUILD_TYPE=Release
|
|
|
|
# vllm.cpp sets no global -march: SIMD tiers are per-file with runtime dispatch,
|
|
# so ONE portable library serves every CPU of the target arch (unlike the
|
|
# ggml-based backends and their avx/avx2/avx512 variant builds).
|
|
UNAME_M := $(shell uname -m)
|
|
|
|
ifeq ($(BUILD_TYPE),cublas)
|
|
# Blackwell-family targets only: other CUDA arches are build-supported
|
|
# upstream but have no runtime-proven fast path. amd64 gets the consumer
|
|
# (120a) + GB10 (121a) fat binary; arm64 CUDA (l4t-style images, DGX
|
|
# Spark) is GB10 only. Triton-AOT GDN cubins are vendored per-arch, no
|
|
# Python needed to consume them.
|
|
ifeq ($(UNAME_M),x86_64)
|
|
# NO -DVLLM_CPP_TRITON on fat builds: the vendored Triton-AOT cubin
|
|
# trees are per-arch and the engine refuses a multi-arch build unless
|
|
# pinned to one tree (unsound for the other arch). The non-AOT GDN
|
|
# path serves the fat binary; single-arch builds keep the cubins.
|
|
#
|
|
# CUDA builds REQUIRE the CUDA 13 toolchain: 12.x nvcc lacks
|
|
# compute_121a (GB10) and its ptxas rejects the sm_120a NVFP4 MMA
|
|
# kernels ("Vector type too large"), so no cuda-12 variant is shipped.
|
|
ifeq ($(CUDA_MAJOR_VERSION),12)
|
|
$(error vllm.cpp needs the CUDA 13 toolchain: CUDA 12.x cannot compile the Blackwell fp4 kernels)
|
|
endif
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=ON "-DVLLM_CPP_CUDA_ARCHITECTURES=120a;121a"
|
|
else
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=ON -DVLLM_CPP_CUDA_ARCHITECTURES=121a -DVLLM_CPP_TRITON=ON
|
|
endif
|
|
else ifeq ($(BUILD_TYPE),vulkan)
|
|
CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF
|
|
else ifeq ($(BUILD_TYPE),metal)
|
|
CMAKE_ARGS+=-DVLLM_CPP_METAL=ON
|
|
else
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF
|
|
endif
|
|
|
|
UNAME_S := $(shell uname -s)
|
|
ifeq ($(UNAME_S),Darwin)
|
|
LIB=libvllm.dylib
|
|
else
|
|
LIB=libvllm.so
|
|
endif
|
|
|
|
sources/vllm.cpp:
|
|
mkdir -p sources/vllm.cpp
|
|
cd sources/vllm.cpp && \
|
|
git init && \
|
|
git remote add origin $(VLLM_CPP_REPO) && \
|
|
git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \
|
|
git checkout FETCH_HEAD
|
|
|
|
$(LIB): sources/vllm.cpp
|
|
mkdir -p build && \
|
|
cd build && \
|
|
cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \
|
|
cmake --build . --config Release -j$(JOBS) --target vllm_shared
|
|
cp -fL build/$(LIB) ./$(LIB)
|
|
|
|
vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB)
|
|
CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./
|
|
|
|
package: vllm-cpp
|
|
bash package.sh
|
|
|
|
build: package
|
|
|
|
clean: purge
|
|
rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp
|
|
|
|
purge:
|
|
rm -rf build
|
|
|
|
.NOTPARALLEL:
|
|
|
|
# The unit specs are pure Go (struct mirrors, option mapping, load
|
|
# validation): no libvllm build is needed. The e2e specs skip unless
|
|
# VLLM_CPP_MODEL points at a real model (then build the lib first).
|
|
test:
|
|
@echo "Running vllm-cpp tests..."
|
|
bash test.sh
|
|
@echo "vllm-cpp tests completed."
|
|
|
|
all: vllm-cpp package
|