mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 18:09:05 -04:00
d49738b59c
Bumps VLLM_CPP_VERSION from 9e1c9025 to eec09bed and turns VLLM_CPP_MLX back on. These two must move together, which is why they are one commit. Upstream now shape-gates the MLX provider to prefill: it declines m < 2, which is exactly the decode GEMV. MLX's steel GEMM wins prefill, 524.5 ms of TTFT against 602 for the native path, but loses decode badly because the provider pays an mx::eval synchronisation and an output memcpy on every call while decode makes about 112 calls per token. Ungated it does both; gated it does only the good half. Measured on an Apple M4 with Qwen3-1.7B-bf16 warm at p=512 g=128: MLX gated to prefill (pin >= 89c46aeb) TTFT 524.5 ms 24.40 tok/s, 99.1% of MLX-LM MLX ungated (older pins) TTFT 537 ms 12.7 tok/s MLX off TTFT 602 ms 23.9 tok/s This branch briefly defaulted the provider off, which was the correct call for an ungated provider at the old pin. The gate is what makes on correct again, so the pin and the flag are coupled: rolling VLLM_CPP_VERSION back before 89c46aeb while leaving MLX on would select the middle row and roughly halve throughput. Both the Makefile comment and the README state that dependency explicitly. The bump also brings six Metal kernels landed upstream since the old pin — mma prefill attention, a vectorised decode V accumulation, vectorised attention staging, a fused qk-norm-RoPE preamble, a simdgroup-per-row softmax and a simdgroup-per-head preamble — which take the non-MLX Metal path from 89.4% to 96.4% of MLX-LM on their own. One caveat, recorded in the README: MLX's GEMM is not bit-identical to the native kernel, so an MLX build produces a different greedy sequence than a non-MLX build. That is a property of the provider rather than of the gate and predates this packaging. Assisted-by: Claude Code:claude-opus-5 [ClaudeCode] Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
168 lines
6.9 KiB
Makefile
168 lines
6.9 KiB
Makefile
CMAKE_ARGS?=
|
|
BUILD_TYPE?=
|
|
NATIVE?=false
|
|
|
|
GOCMD?=go
|
|
GO_TAGS?=
|
|
# nproc doesn't exist on the macOS runners: an empty JOBS turns `-j$(JOBS)`
|
|
# into bare `-j` (unlimited clang jobs), which swap-thrashes the 3-core Mac
|
|
# until the 6h GHA timeout. Fall back to sysctl there, then to a constant.
|
|
JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
|
|
|
# vllm.cpp version
|
|
VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
|
|
VLLM_CPP_VERSION?=eec09bed5a03457837b499781c23d8e44f106813
|
|
|
|
# MLX GEMM provider (darwin/metal only; see the metal branch below for why).
|
|
# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun
|
|
# metal`, i.e. a full Xcode the macOS runners do not have, while the wheel ships
|
|
# include/, lib/libmlx.dylib and the compiled mlx.metallib ready to link.
|
|
#
|
|
# DEFAULT ON, but ONLY because VLLM_CPP_VERSION above is pinned at or past
|
|
# vllm.cpp 89c46aeb, which SHAPE-GATES the provider to prefill. The ordering is
|
|
# load-bearing, not incidental:
|
|
#
|
|
# pin >= 89c46aeb, MLX on -> 99.1% of MLX-LM (gated: prefill only)
|
|
# pin < 89c46aeb, MLX on -> ~51% (ungated: it also takes decode)
|
|
#
|
|
# MLX's steel GEMM wins prefill (537 ms TTFT against 602) and loses decode badly,
|
|
# because the provider pays an mx::eval sync plus an output memcpy per call and
|
|
# decode makes ~112 calls per TOKEN. Ungated it does both; gated it does only the
|
|
# good half. So if this pin is ever moved BACKWARDS, this default must go with it.
|
|
VLLM_CPP_MLX?=on
|
|
MLX_VERSION?=0.29.3
|
|
MLX_VENV?=$(abspath ./mlx-venv)
|
|
# Resolved lazily (recursive `=`, not `:=`): the glob only matches once the venv
|
|
# target has run, and the interpreter version in the path varies per runner.
|
|
MLX_ROOT=$(shell echo $(MLX_VENV)/lib/python*/site-packages/mlx)
|
|
|
|
# The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
|
|
# server, examples and tests of the engine are never built here.
|
|
CMAKE_ARGS+=-DVLLM_CPP_SERVER=OFF -DVLLM_CPP_BUILD_TESTS=OFF -DVLLM_CPP_BUILD_EXAMPLES=OFF
|
|
CMAKE_ARGS+=-DCMAKE_BUILD_TYPE=Release
|
|
|
|
# vllm.cpp sets no global -march: SIMD tiers are per-file with runtime dispatch,
|
|
# so ONE portable library serves every CPU of the target arch (unlike the
|
|
# ggml-based backends and their avx/avx2/avx512 variant builds).
|
|
UNAME_M := $(shell uname -m)
|
|
|
|
ifeq ($(BUILD_TYPE),cublas)
|
|
# Blackwell-family targets only: other CUDA arches are build-supported
|
|
# upstream but have no runtime-proven fast path. amd64 gets the consumer
|
|
# (120a) + GB10 (121a) fat binary; arm64 CUDA (l4t-style images, DGX
|
|
# Spark) is GB10 only. Triton-AOT GDN cubins are vendored per-arch, no
|
|
# Python needed to consume them.
|
|
ifeq ($(UNAME_M),x86_64)
|
|
# NO -DVLLM_CPP_TRITON on fat builds: the vendored Triton-AOT cubin
|
|
# trees are per-arch and the engine refuses a multi-arch build unless
|
|
# pinned to one tree (unsound for the other arch). The non-AOT GDN
|
|
# path serves the fat binary; single-arch builds keep the cubins.
|
|
#
|
|
# CUDA builds REQUIRE the CUDA 13 toolchain: 12.x nvcc lacks
|
|
# compute_121a (GB10) and its ptxas rejects the sm_120a NVFP4 MMA
|
|
# kernels ("Vector type too large"), so no cuda-12 variant is shipped.
|
|
ifeq ($(CUDA_MAJOR_VERSION),12)
|
|
$(error vllm.cpp needs the CUDA 13 toolchain: CUDA 12.x cannot compile the Blackwell fp4 kernels)
|
|
endif
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=ON "-DVLLM_CPP_CUDA_ARCHITECTURES=120a;121a"
|
|
else
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=ON -DVLLM_CPP_CUDA_ARCHITECTURES=121a -DVLLM_CPP_TRITON=ON
|
|
endif
|
|
else ifeq ($(BUILD_TYPE),vulkan)
|
|
CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF
|
|
else ifeq ($(BUILD_TYPE),metal)
|
|
CMAKE_ARGS+=-DVLLM_CPP_METAL=ON
|
|
# The optional MLX GEMM provider. vllm.cpp keeps it OFF by default because it
|
|
# is a ~19 MB libmlx.dylib plus a ~105 MB mlx.metallib, and upstream's
|
|
# position is that it must earn that cost by measurement. It does, on the
|
|
# only hardware this build targets: measured on an Apple M4 against the
|
|
# native MSL GEMM in the SAME binary (arms toggled by
|
|
# VT_OP_PROVIDER_DISABLE=mlx), Qwen3-1.7B-bf16 p=512 g=128, it is 1.5x to
|
|
# 2.2x aggregate throughput and 2x to 3x faster TTFT, at equal peak memory
|
|
# and bit-identical output on every parity shape. See vllm.cpp
|
|
# docs/BENCHMARKS.md "MLX GEMM provider A/B on Apple M4".
|
|
#
|
|
# MLX delegates the dense GEMM ONLY: kPagedAttention stays vllm.cpp's own
|
|
# kernel, because MLX has no paged-KV primitive at all.
|
|
#
|
|
# Set VLLM_CPP_MLX=off for a Metal build without it (smaller image, slower).
|
|
ifeq ($(VLLM_CPP_MLX),on)
|
|
MLX_ENABLED=1
|
|
endif
|
|
else
|
|
CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF
|
|
endif
|
|
|
|
UNAME_S := $(shell uname -s)
|
|
ifeq ($(UNAME_S),Darwin)
|
|
LIB=libvllm.dylib
|
|
else
|
|
LIB=libvllm.so
|
|
endif
|
|
|
|
sources/vllm.cpp:
|
|
mkdir -p sources/vllm.cpp
|
|
cd sources/vllm.cpp && \
|
|
git init && \
|
|
git remote add origin $(VLLM_CPP_REPO) && \
|
|
git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \
|
|
git checkout FETCH_HEAD
|
|
|
|
ifeq ($(MLX_ENABLED),1)
|
|
# A stamp FILE, not a phony target: a phony prerequisite is always "newer" than
|
|
# $(LIB) and would re-link libvllm on every invocation. Keyed on the version so
|
|
# a MLX_VERSION bump reinstalls instead of silently reusing the old wheel.
|
|
MLX_STAMP=$(MLX_VENV)/.mlx-$(MLX_VERSION).stamp
|
|
MLX_CMAKE_ARGS=-DVLLM_CPP_MLX=ON -DMLX_ROOT=$(MLX_ROOT)
|
|
|
|
$(MLX_STAMP):
|
|
@if [ ! -x "$(MLX_VENV)/bin/pip" ]; then \
|
|
python3 -m venv "$(MLX_VENV)" || { echo "vllm-cpp: python3 with venv is required to build the MLX provider; pass VLLM_CPP_MLX=off to build Metal without it" >&2; exit 1; }; \
|
|
fi
|
|
"$(MLX_VENV)"/bin/pip install --quiet --disable-pip-version-check "mlx==$(MLX_VERSION)"
|
|
@# Resolved in the SHELL, not by $(MLX_ROOT): make expands a whole recipe
|
|
@# before running its first line, so the glob would still be unmatched here.
|
|
@# Every later use (the cmake args, package.sh) expands after this target has
|
|
@# completed, where $(MLX_ROOT) does resolve.
|
|
@root=$$(echo "$(MLX_VENV)"/lib/python*/site-packages/mlx); \
|
|
test -f "$$root/lib/libmlx.dylib" -a -f "$$root/include/mlx/array.h" || \
|
|
{ echo "vllm-cpp: mlx==$(MLX_VERSION) did not provide lib/libmlx.dylib + include/mlx/array.h under $$root" >&2; exit 1; }
|
|
touch $@
|
|
else
|
|
MLX_STAMP=
|
|
MLX_CMAKE_ARGS=
|
|
endif
|
|
|
|
$(LIB): sources/vllm.cpp $(MLX_STAMP)
|
|
mkdir -p build && \
|
|
cd build && \
|
|
cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \
|
|
cmake --build . --config Release -j$(JOBS) --target vllm_shared
|
|
cp -fL build/$(LIB) ./$(LIB)
|
|
|
|
vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB)
|
|
CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./
|
|
|
|
package: vllm-cpp
|
|
MLX_ROOT="$(MLX_ROOT)" bash package.sh
|
|
|
|
build: package
|
|
|
|
clean: purge
|
|
rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp "$(MLX_VENV)"
|
|
|
|
purge:
|
|
rm -rf build
|
|
|
|
.NOTPARALLEL:
|
|
|
|
# The unit specs are pure Go (struct mirrors, option mapping, load
|
|
# validation): no libvllm build is needed. The e2e specs skip unless
|
|
# VLLM_CPP_MODEL points at a real model (then build the lib first).
|
|
test:
|
|
@echo "Running vllm-cpp tests..."
|
|
bash test.sh
|
|
@echo "vllm-cpp tests completed."
|
|
|
|
all: vllm-cpp package
|