mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
40e7c76ec2
The pin sat at f384edcd while vllm.cpp main moved a long way. The ABI is unchanged at v10 and both POD structs are field-identical to the pinned commit (verified by diffing vllm_model_params and vllm_sampling_params across the range), so the Go mirror needs no edit and this is a clean bump. What it picks up matters for this backend: - MTP speculative decoding from a GGUF target, gated end to end on GPU. - DFlash speculative decoding with a GGUF draft AND a GGUF target. - NVFP4 GGUF: dequant, plus a native fp4 compute path for dense and full-attention projections. On the 27B that closed a cross-container divergence entirely (the GGUF and safetensors builds of the same quantization run now emit identical tokens) and halved peak RSS. - A real engine fix: the GDN speculative state gather/scatter was mis-striding the widened conv row, so speculation silently corrupted the target's own recurrent state on CPU. Docs corrected accordingly. The section previously told users that mtp and dflash are rejected on a .gguf target and called it a gap in the engine's GGUF loader. That is no longer true, and leaving it would send people to safetensors for no reason. A head-less GGUF is still refused, and the text now says so with the actual cause. The real-library ABI handshake was re-run against a libvllm.so built at the exact pinned commit rather than a stale one: 43 specs pass, reported ABI 10. make lint clean. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude Code:claude-opus-5 [ClaudeCode]
103 lines
3.5 KiB
Makefile
103 lines
3.5 KiB
Makefile
CMAKE_ARGS?=
|
|
BUILD_TYPE?=
|
|
NATIVE?=false
|
|
|
|
GOCMD?=go
|
|
GO_TAGS?=
|
|
# nproc doesn't exist on the macOS runners: an empty JOBS turns `-j$(JOBS)`
|
|
# into bare `-j` (unlimited clang jobs), which swap-thrashes the 3-core Mac
|
|
# until the 6h GHA timeout. Fall back to sysctl there, then to a constant.
|
|
JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
|
|
|
# vllm.cpp version
|
|
VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
|
|
VLLM_CPP_VERSION?=6d4fb5e7ae13eb8a30fdd3f5e8df8a3d8a83143b
|
|
|
|
# The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
|
|
# server, examples and tests of the engine are never built here.
|
|
CMAKE_ARGS+=-DVLLM_CPP_SERVER=OFF -DVLLM_CPP_BUILD_TESTS=OFF -DVLLM_CPP_BUILD_EXAMPLES=OFF
|
|
CMAKE_ARGS+=-DCMAKE_BUILD_TYPE=Release
|
|
|
|
# vllm.cpp sets no global -march: SIMD tiers are per-file with runtime dispatch,
|
|
# so ONE portable library serves every CPU of the target arch (unlike the
|
|
# ggml-based backends and their avx/avx2/avx512 variant builds).
|
|
UNAME_M := $(shell uname -m)
|
|
|
|
ifeq ($(BUILD_TYPE),cublas)
|
|
# Blackwell-family targets only: other CUDA arches are build-supported
|
|
# upstream but have no runtime-proven fast path. amd64 gets the consumer
|
|
# (120a) + GB10 (121a) fat binary; arm64 CUDA (l4t-style images, DGX
|
|
# Spark) is GB10 only. Triton-AOT GDN cubins are vendored per-arch, no
|
|
# Python needed to consume them.
|
|
ifeq ($(UNAME_M),x86_64)
|
|
# NO -DVLLM_CPP_TRITON on fat builds: the vendored Triton-AOT cubin
|
|
# trees are per-arch and the engine refuses a multi-arch build unless
|
|
# pinned to one tree (unsound for the other arch). The non-AOT GDN
|
|
# path serves the fat binary; single-arch builds keep the cubins.
|
|
#
|
|
# CUDA builds REQUIRE the CUDA 13 toolchain: 12.x nvcc lacks
|
|
# compute_121a (GB10) and its ptxas rejects the sm_120a NVFP4 MMA
|
|
# kernels ("Vector type too large"), so no cuda-12 variant is shipped.
|
|
ifeq ($(CUDA_MAJOR_VERSION),12)
|
|
$(error vllm.cpp needs the CUDA 13 toolchain: CUDA 12.x cannot compile the Blackwell fp4 kernels)
|
|
endif
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=ON "-DVLLM_CPP_CUDA_ARCHITECTURES=120a;121a"
|
|
else
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=ON -DVLLM_CPP_CUDA_ARCHITECTURES=121a -DVLLM_CPP_TRITON=ON
|
|
endif
|
|
else ifeq ($(BUILD_TYPE),vulkan)
|
|
CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF
|
|
else ifeq ($(BUILD_TYPE),metal)
|
|
CMAKE_ARGS+=-DVLLM_CPP_METAL=ON
|
|
else
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF
|
|
endif
|
|
|
|
UNAME_S := $(shell uname -s)
|
|
ifeq ($(UNAME_S),Darwin)
|
|
LIB=libvllm.dylib
|
|
else
|
|
LIB=libvllm.so
|
|
endif
|
|
|
|
sources/vllm.cpp:
|
|
mkdir -p sources/vllm.cpp
|
|
cd sources/vllm.cpp && \
|
|
git init && \
|
|
git remote add origin $(VLLM_CPP_REPO) && \
|
|
git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \
|
|
git checkout FETCH_HEAD
|
|
|
|
$(LIB): sources/vllm.cpp
|
|
mkdir -p build && \
|
|
cd build && \
|
|
cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \
|
|
cmake --build . --config Release -j$(JOBS) --target vllm_shared
|
|
cp -fL build/$(LIB) ./$(LIB)
|
|
|
|
vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB)
|
|
CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./
|
|
|
|
package: vllm-cpp
|
|
bash package.sh
|
|
|
|
build: package
|
|
|
|
clean: purge
|
|
rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp
|
|
|
|
purge:
|
|
rm -rf build
|
|
|
|
.NOTPARALLEL:
|
|
|
|
# The unit specs are pure Go (struct mirrors, option mapping, load
|
|
# validation): no libvllm build is needed. The e2e specs skip unless
|
|
# VLLM_CPP_MODEL points at a real model (then build the lib first).
|
|
test:
|
|
@echo "Running vllm-cpp tests..."
|
|
bash test.sh
|
|
@echo "vllm-cpp tests completed."
|
|
|
|
all: vllm-cpp package
|