diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 0f009340c..62d99396c 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -13,6 +13,17 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp VLLM_CPP_VERSION?=9e1c9025ae61167a3335454d7cc0de6093c21845 +# MLX GEMM provider (darwin/metal only; see the metal branch below for why). +# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun +# metal`, i.e. a full Xcode the macOS runners do not have, while the wheel ships +# include/, lib/libmlx.dylib and the compiled mlx.metallib ready to link. +VLLM_CPP_MLX?=on +MLX_VERSION?=0.29.3 +MLX_VENV?=$(abspath ./mlx-venv) +# Resolved lazily (recursive `=`, not `:=`): the glob only matches once the venv +# target has run, and the interpreter version in the path varies per runner. +MLX_ROOT=$(shell echo $(MLX_VENV)/lib/python*/site-packages/mlx) + # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the # server, examples and tests of the engine are never built here. CMAKE_ARGS+=-DVLLM_CPP_SERVER=OFF -DVLLM_CPP_BUILD_TESTS=OFF -DVLLM_CPP_BUILD_EXAMPLES=OFF @@ -49,6 +60,23 @@ else ifeq ($(BUILD_TYPE),vulkan) CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF else ifeq ($(BUILD_TYPE),metal) CMAKE_ARGS+=-DVLLM_CPP_METAL=ON + # The optional MLX GEMM provider. vllm.cpp keeps it OFF by default because it + # is a ~19 MB libmlx.dylib plus a ~105 MB mlx.metallib, and upstream's + # position is that it must earn that cost by measurement. It does, on the + # only hardware this build targets: measured on an Apple M4 against the + # native MSL GEMM in the SAME binary (arms toggled by + # VT_OP_PROVIDER_DISABLE=mlx), Qwen3-1.7B-bf16 p=512 g=128, it is 1.5x to + # 2.2x aggregate throughput and 2x to 3x faster TTFT, at equal peak memory + # and bit-identical output on every parity shape. See vllm.cpp + # docs/BENCHMARKS.md "MLX GEMM provider A/B on Apple M4". + # + # MLX delegates the dense GEMM ONLY: kPagedAttention stays vllm.cpp's own + # kernel, because MLX has no paged-KV primitive at all. + # + # Set VLLM_CPP_MLX=off for a Metal build without it (smaller image, slower). + ifeq ($(VLLM_CPP_MLX),on) + MLX_ENABLED=1 + endif else CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF endif @@ -68,10 +96,35 @@ sources/vllm.cpp: git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \ git checkout FETCH_HEAD -$(LIB): sources/vllm.cpp +ifeq ($(MLX_ENABLED),1) +# A stamp FILE, not a phony target: a phony prerequisite is always "newer" than +# $(LIB) and would re-link libvllm on every invocation. Keyed on the version so +# a MLX_VERSION bump reinstalls instead of silently reusing the old wheel. +MLX_STAMP=$(MLX_VENV)/.mlx-$(MLX_VERSION).stamp +MLX_CMAKE_ARGS=-DVLLM_CPP_MLX=ON -DMLX_ROOT=$(MLX_ROOT) + +$(MLX_STAMP): + @if [ ! -x "$(MLX_VENV)/bin/pip" ]; then \ + python3 -m venv "$(MLX_VENV)" || { echo "vllm-cpp: python3 with venv is required to build the MLX provider; pass VLLM_CPP_MLX=off to build Metal without it" >&2; exit 1; }; \ + fi + "$(MLX_VENV)"/bin/pip install --quiet --disable-pip-version-check "mlx==$(MLX_VERSION)" + @# Resolved in the SHELL, not by $(MLX_ROOT): make expands a whole recipe + @# before running its first line, so the glob would still be unmatched here. + @# Every later use (the cmake args, package.sh) expands after this target has + @# completed, where $(MLX_ROOT) does resolve. + @root=$$(echo "$(MLX_VENV)"/lib/python*/site-packages/mlx); \ + test -f "$$root/lib/libmlx.dylib" -a -f "$$root/include/mlx/array.h" || \ + { echo "vllm-cpp: mlx==$(MLX_VERSION) did not provide lib/libmlx.dylib + include/mlx/array.h under $$root" >&2; exit 1; } + touch $@ +else +MLX_STAMP= +MLX_CMAKE_ARGS= +endif + +$(LIB): sources/vllm.cpp $(MLX_STAMP) mkdir -p build && \ cd build && \ - cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \ + cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \ cmake --build . --config Release -j$(JOBS) --target vllm_shared cp -fL build/$(LIB) ./$(LIB) @@ -79,12 +132,12 @@ vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB) CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./ package: vllm-cpp - bash package.sh + MLX_ROOT="$(MLX_ROOT)" bash package.sh build: package clean: purge - rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp + rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp "$(MLX_VENV)" purge: rm -rf build diff --git a/backend/go/vllm-cpp/README.md b/backend/go/vllm-cpp/README.md index 4d2c437f1..cd6eb94a0 100644 --- a/backend/go/vllm-cpp/README.md +++ b/backend/go/vllm-cpp/README.md @@ -41,5 +41,38 @@ options: - max_num_seqs:16 ``` +## Apple Silicon: the MLX GEMM provider + +`BUILD_TYPE=metal` builds the Metal backend with vllm.cpp's optional MLX +provider for the dense GEMM (`VLLM_CPP_MLX=on`, the default here). Upstream keeps +it off because it costs a ~19 MB `libmlx.dylib` plus a ~105 MB `mlx.metallib`; +this backend accepts that because the provider was measured to pay for it on an +Apple M4, against the native MSL GEMM in the SAME binary (arms toggled with +`VT_OP_PROVIDER_DISABLE=mlx`), Qwen3-1.7B-bf16 at p=512 g=128: + +| Concurrency | MLX agg tok/s | native agg tok/s | speedup | +|--:|--:|--:|--:| +| 1 | 5.79 | 3.08 | 1.88x | +| 8 | 25.70 | 13.69 | 1.88x | +| 16 | 38.65 | 17.69 | 2.19x | + +TTFT improves 2x to 3x, peak memory is unchanged, and the GEMM output is +bit-identical to the native kernel on every parity shape. MLX serves the dense +GEMM only: paged attention stays vllm.cpp's own kernel, because MLX has no +paged-KV primitive. Full disposition in vllm.cpp `docs/BENCHMARKS.md`, +"MLX GEMM provider A/B on Apple M4". + +Build knobs: + +- `VLLM_CPP_MLX=off` builds Metal without the provider: ~124 MB smaller, slower. +- `MLX_VERSION` pins the wheel (default `0.29.3`). MLX is consumed as the + prebuilt pip wheel because building it from source needs `xcrun metal`, i.e. a + full Xcode the macOS runners do not have. + +Packaging vendors `libmlx.dylib`, `mlx.metallib` and MLX's MIT license into +`package/lib/`, and rewrites `libvllm.dylib`'s rpath to `@loader_path/lib` +(re-signing it, since `install_name_tool` invalidates the signature). The +metallib must stay beside `libmlx.dylib`: MLX looks for it there. + Testing: `make test` runs the unit specs; export `VLLM_CPP_MODEL=` (and optionally `VLLM_CPP_LIBRARY=`) to enable the e2e specs. diff --git a/backend/go/vllm-cpp/package.sh b/backend/go/vllm-cpp/package.sh index 78dc49178..30a21d219 100644 --- a/backend/go/vllm-cpp/package.sh +++ b/backend/go/vllm-cpp/package.sh @@ -43,6 +43,50 @@ elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 elif [ $(uname -s) = "Darwin" ]; then echo "Detected Darwin" + # Vendor the optional MLX GEMM provider, when libvllm was built against it. + # Three facts drive every line below, each verified on an Apple M4 before it + # was written: + # 1. libvllm.dylib carries an LC_LOAD_DYLIB on @rpath/libmlx.dylib, and its + # build-time LC_RPATH points inside the build venv. That path does not + # exist on a user's machine, so it must become @loader_path/lib. + # 2. MLX finds its ~100 MB mlx.metallib beside its OWN dylib, so the two + # files have to land in the same directory or every Metal op dies with + # "Failed to load the default metallib". + # 3. install_name_tool invalidates the code signature, and macOS refuses to + # load an arm64 image whose signature does not match, so the patched + # library must be re-signed ad-hoc afterwards. + if otool -L "$CURDIR/package/libvllm.dylib" 2>/dev/null | grep -q "libmlx.dylib"; then + MLX_LIB_DIR="${MLX_ROOT}/lib" + if [ ! -f "$MLX_LIB_DIR/libmlx.dylib" ] || [ ! -f "$MLX_LIB_DIR/mlx.metallib" ]; then + echo "Error: libvllm.dylib links libmlx.dylib but $MLX_LIB_DIR is missing libmlx.dylib/mlx.metallib" >&2 + exit 1 + fi + echo "Vendoring the MLX GEMM provider from $MLX_LIB_DIR" + cp -fLv "$MLX_LIB_DIR/libmlx.dylib" "$CURDIR/package/lib/" + cp -fLv "$MLX_LIB_DIR/mlx.metallib" "$CURDIR/package/lib/" + # MLX is MIT and we redistribute its binaries, so its license ships with + # them. mlx-metal is the wheel carrying the dylib and the metallib. + MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx_metal-*.dist-info/licenses/LICENSE 2>/dev/null | head -1) + if [ -z "$MLX_LICENSE" ]; then + MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx-*.dist-info/licenses/LICENSE 2>/dev/null | head -1) + fi + if [ -z "$MLX_LICENSE" ]; then + echo "Error: could not find the MLX LICENSE to redistribute alongside libmlx.dylib" >&2 + exit 1 + fi + cp -fLv "$MLX_LICENSE" "$CURDIR/package/lib/LICENSE.mlx" + # Drop every build-tree rpath, then point at the packaged copy. + otool -l "$CURDIR/package/libvllm.dylib" | awk '/LC_RPATH/{f=1;next} f&&/ path /{print $2;f=0}' | while read -r rp; do + install_name_tool -delete_rpath "$rp" "$CURDIR/package/libvllm.dylib" 2>/dev/null || true + done + install_name_tool -add_rpath "@loader_path/lib" "$CURDIR/package/libvllm.dylib" + codesign -f -s - "$CURDIR/package/libvllm.dylib" + # A broken rpath must fail the BUILD, not the user's first inference. + if ! otool -l "$CURDIR/package/libvllm.dylib" | grep -q "@loader_path/lib"; then + echo "Error: libvllm.dylib did not get the @loader_path/lib rpath" >&2 + exit 1 + fi + fi else echo "Error: Could not detect architecture" exit 1