diff --git a/Makefile b/Makefile index 9d4109b1d..51f61fc63 100644 --- a/Makefile +++ b/Makefile @@ -245,6 +245,8 @@ test-ci-scripts: PYTHON_HELPER_TESTS?=python_utils_test vllm_utils_test model_utils_test mlx_utils_test parent_watch_test temp_utils_test test-python-helpers: cd backend/python/common && python3 -m unittest $(PYTHON_HELPER_TESTS) + bash backend/python/vllm-omni/test_install.sh + python3 -m unittest discover -s backend/python/vllm-omni -p 'test_install.py' ## Runs the core suite ($(TEST_PATHS)) with statement-coverage instrumentation ## and writes a merged profile to $(COVERAGE_PROFILE). Deliberately omits diff --git a/backend/python/vllm-omni/Makefile b/backend/python/vllm-omni/Makefile index aaeb3188a..aaa3ffbcd 100644 --- a/backend/python/vllm-omni/Makefile +++ b/backend/python/vllm-omni/Makefile @@ -21,3 +21,8 @@ protogen-clean: .PHONY: clean clean: protogen-clean rm -rf venv __pycache__ + +# No model downloads, Python packages, or GPU required. +.PHONY: test-install +test-install: + bash test_install.sh diff --git a/backend/python/vllm-omni/backend.py b/backend/python/vllm-omni/backend.py index 278a0e248..f1bd59949 100644 --- a/backend/python/vllm-omni/backend.py +++ b/backend/python/vllm-omni/backend.py @@ -40,7 +40,7 @@ from vllm_utils import parse_options, messages_to_dicts, setup_parsers from vllm_omni.entrypoints.omni import Omni from vllm_omni.outputs import OmniRequestOutput from vllm_omni.diffusion.data import DiffusionParallelConfig -from vllm_omni.utils.platform_utils import detect_device_type, is_npu +from vllm_omni.platforms import current_platform from vllm import SamplingParams from diffusers.utils import export_to_video @@ -204,8 +204,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # Add diffusion-specific parameters (image/video models) if self.model_type in ["image", "video"]: omni_kwargs.update({ - "vae_use_slicing": is_npu(), - "vae_use_tiling": is_npu(), + "vae_use_slicing": current_platform.is_npu(), + "vae_use_tiling": current_platform.is_npu(), "cache_backend": cache_backend, "cache_config": cache_config, "parallel_config": parallel_config, @@ -284,7 +284,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # Create generator if seed provided generator = None if seed: - device = detect_device_type() + device = current_platform.device_type generator = torch.Generator(device=device).manual_seed(seed) # Handle image input for image editing @@ -364,7 +364,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # Create generator generator = None if seed: - device = detect_device_type() + device = current_platform.device_type generator = torch.Generator(device=device).manual_seed(seed) # Handle image input for image-to-video diff --git a/backend/python/vllm-omni/install.sh b/backend/python/vllm-omni/install.sh index fea6855b4..f7ef749e8 100755 --- a/backend/python/vllm-omni/install.sh +++ b/backend/python/vllm-omni/install.sh @@ -5,91 +5,93 @@ PYTHON_VERSION="3.12" PYTHON_PATCH="12" PY_STANDALONE_TAG="20251120" -backend_dir=$(dirname $0) -if [ -d $backend_dir/common ]; then - source $backend_dir/common/libbackend.sh +backend_dir=$(dirname "$0") +if [ -d "$backend_dir/common" ]; then + source "$backend_dir/common/libbackend.sh" else - source $backend_dir/../common/libbackend.sh -fi - -# Handle l4t build profiles (Python 3.12, pip fallback) if needed. -# Since PyTorch 2.11 (April 2026) PyPI ships aarch64 + cu130 manylinux wheels -# directly for torch/torchvision/torchaudio and an aarch64 vllm wheel pinned -# to that torch, so the jetson-ai-lab mirror is no longer needed. -# https://pytorch.org/blog/vllm-and-pytorch-work-together-to-improve-the-developer-experience-on-aarch64/ -if [ "x${BUILD_PROFILE}" == "xl4t13" ]; then - PYTHON_VERSION="3.12" - PYTHON_PATCH="12" - PY_STANDALONE_TAG="20251120" + source "$backend_dir/../common/libbackend.sh" fi if [ "x${BUILD_PROFILE}" == "xl4t12" ]; then USE_PIP=true fi -# Install base requirements first -installRequirements - -# Install vllm based on build type. vllm-omni tracks vllm master from -# source (cloned below) so we leave the upstream vllm dependency unpinned -# — vllm 0.19+ ships cu130 wheels by default, which is what we want for -# cublas13. Older cuda12/rocm/cpu paths still resolve a compatible wheel -# from the relevant channel. +# Omni and vLLM share internal APIs. Keep matched releases instead of resolving +# either project from main/latest. CUDA 13 needs the newer aarch64 wheels. +VLLM_VERSION=0.14.0 +# vllm-omni v0.14.0 +OMNI_REVISION=ed89c8b0436999e9210f11363f4eb512330a9dfa +engine_flags=() if [ "x${BUILD_TYPE}" == "xhipblas" ]; then - # ROCm - if [ "x${USE_PIP}" == "xtrue" ]; then - pip install vllm==0.14.0 --extra-index-url https://wheels.vllm.ai/rocm/0.14.0/rocm700 - else - uv pip install vllm==0.14.0 --extra-index-url https://wheels.vllm.ai/rocm/0.14.0/rocm700 - fi + engine_flags=(--extra-index-url https://wheels.vllm.ai/rocm/0.14.0/rocm700) elif [ "x${BUILD_PROFILE}" == "xcublas13" ] || [ "x${BUILD_PROFILE}" == "xl4t13" ]; then - # cublas13 (x86_64) and l4t13 (aarch64) both pull vllm from PyPI now: - # vllm 0.19+ defaults to cu130 wheels on x86_64 and vllm 0.20+ ships an - # aarch64 manylinux wheel pinned to torch==2.11.0. No extra index needed - # in either case. - if [ "x${USE_PIP}" == "xtrue" ]; then - pip install vllm --torch-backend=auto - else - uv pip install vllm --torch-backend=auto - fi -elif [ "x${BUILD_TYPE}" == "xcublas" ] || [ "x${BUILD_TYPE}" == "x" ]; then - # cuda12 / CPU — keep the 0.14.0 pin for compatibility with the existing - # cuda12 vllm-omni image; bumping should be its own change. - if [ "x${USE_PIP}" == "xtrue" ]; then - pip install vllm==0.14.0 --torch-backend=auto - else - uv pip install vllm==0.14.0 --torch-backend=auto - fi -else + VLLM_VERSION=0.20.0 + # vllm-omni v0.20.0 still accepts the stage_configs_path model option. + OMNI_REVISION=4a24a517abc7769b1399ded594558a3fe8269872 +elif [ "x${BUILD_TYPE}" != "xcublas" ] && [ "x${BUILD_TYPE}" != "x" ]; then echo "Unsupported build type: ${BUILD_TYPE}" >&2 exit 1 fi -# Clone and install vllm-omni from source -if [ ! -d vllm-omni ]; then - git clone https://github.com/vllm-project/vllm-omni.git -fi - -cd vllm-omni/ - -# fa3-fwd ships no aarch64 wheels and there is no source distribution, so on -# aarch64 (e.g. l4t13 / SBSA cu130) the upstream requirements/cuda.txt is -# unsatisfiable. Drop it before resolving — vllm-omni does not hard-require -# the fused FA3 kernel at import time on Jetson/SBSA targets. -if [ "$(uname -m)" = "aarch64" ] && [ -f requirements/cuda.txt ]; then - sed -i '/^fa3-fwd[[:space:]]*==/d' requirements/cuda.txt -fi +installRequirements if [ "x${USE_PIP}" == "xtrue" ]; then - pip install ${EXTRA_PIP_INSTALL_FLAGS:-} -e . + installer=(pip install) else - uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} -e . + installer=(uv pip install) + if [ "x${BUILD_TYPE}" != "xhipblas" ]; then + engine_flags+=(--torch-backend=auto) + fi fi +"${installer[@]}" "vllm==${VLLM_VERSION}" "${engine_flags[@]}" -cd .. +# Use a fresh checkout so a previous build cannot leave a stale or modified +# source tree. The installed wheel must survive removal of this directory and +# relocation of the packaged backend's virtual environment. +omni_source=$(mktemp -d) +trap 'rm -rf "$omni_source"' EXIT +( + cd "$omni_source" + git init + git fetch --depth 1 https://github.com/vllm-project/vllm-omni.git "$OMNI_REVISION" + git checkout --detach FETCH_HEAD + # Preserve release metadata for setuptools-scm in the shallow checkout. + git tag "v${VLLM_VERSION}" -# vllm-omni lands after installRequirements has already generated the protobuf -# stubs, and it re-resolves the protobuf runtime as it installs. Regenerate now -# that the dependency set is final, so the stubs cannot be newer than the runtime -# that ships. Same failure mode as mudler/LocalAI#10718. + # FA3 publishes no aarch64 wheel. These releases keep this optional CUDA + # kernel dependency in different files; neither needs it to import on ARM. + if [ "$(uname -m)" = "aarch64" ]; then + if [ -f requirements/cuda.txt ]; then + sed -i '/^fa3-fwd[[:space:]]*==/d' requirements/cuda.txt + fi + sed -i '/^[[:space:]]*"fa3-fwd==/d' pyproject.toml + fi + + # v0.14 only declares generic stage configs as package data. Preserve the + # platform YAML files too, or ROCm silently uses generic batching defaults. + printf '\nrecursive-include vllm_omni *.yaml\n' >> MANIFEST.in + + # Retain the engine pin while resolving Omni's dependencies too. + "${installer[@]}" ${EXTRA_PIP_INSTALL_FLAGS:-} . "vllm==${VLLM_VERSION}" "${engine_flags[@]}" + + # Check wheel resources without importing GPU-dependent modules. Verify + # every source YAML, so future upstream packaging changes fail the build. + python -I - <<'PY' +from importlib.metadata import distribution +from pathlib import Path + +dist = distribution("vllm-omni") +installed = {str(path) for path in dist.files or []} +resources = list(Path("vllm_omni").rglob("*.yaml")) +if not resources: + raise RuntimeError("vllm-omni source contains no YAML resources") +for source in resources: + target = Path(dist.locate_file(str(source))) + if (str(source) not in installed or not target.is_file() + or target.read_bytes() != source.read_bytes()): + raise RuntimeError(f"vllm-omni wheel omits or changes {source}") +PY +) + +# Omni can change the protobuf runtime after the initial stub generation. runProtogen diff --git a/backend/python/vllm-omni/test_install.py b/backend/python/vllm-omni/test_install.py new file mode 100644 index 000000000..1c2ad9715 --- /dev/null +++ b/backend/python/vllm-omni/test_install.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: MIT +"""CPU-only install and platform regressions: python -m unittest discover -s backend/python/vllm-omni -p 'test_install.py'.""" +import ast +import contextlib +import io +import os +from pathlib import Path +import sys +import traceback +import types +import unittest +from unittest.mock import patch + +BACKEND = Path(__file__).resolve().parent + + +class PlatformTests(unittest.TestCase): + def test_backend_import_uses_supported_platform_api(self): + tree = ast.parse((BACKEND / 'backend.py').read_text()) + imports = [node for node in tree.body if isinstance(node, ast.ImportFrom) + and node.module.startswith('vllm_omni')] + modules = {} + platform = types.SimpleNamespace(device_type='cuda', is_npu=lambda: False) + for node in imports: + if node.module == 'vllm_omni.utils.platform_utils': + continue # Removed upstream; both pinned releases provide platforms instead. + module = types.ModuleType(node.module) + for alias in node.names: + setattr(module, alias.name, platform if alias.name == 'current_platform' else object) + modules[node.module] = module + namespace = {} + with patch.dict('sys.modules', modules): + exec(compile(ast.Module(body=imports, type_ignores=[]), '', 'exec'), namespace) + self.assertIs(namespace['current_platform'], platform) + + def load_model(self, model_type, options=None, npu=False): + tree = ast.parse((BACKEND / 'backend.py').read_text()) + servicer = next(node for node in tree.body if isinstance(node, ast.ClassDef) + and node.name == 'BackendServicer') + method = next(node for node in servicer.body if isinstance(node, ast.FunctionDef) + and node.name == 'LoadModel') + calls = [] + + # Both pinned releases accept stage_configs_path and these constructor + # options. Run the production method, stubbing only its dependencies. + def omni(model, *, stage_configs_path=None, log_stats=False, + stage_init_timeout=300, **kwargs): + calls.append(dict(model=model, stage_configs_path=stage_configs_path, + log_stats=log_stats, stage_init_timeout=stage_init_timeout, + **kwargs)) + return object() + + namespace = dict( + os=os, sys=sys, traceback=traceback, + torch=types.SimpleNamespace(cuda=types.SimpleNamespace(is_available=lambda: True)), + current_platform=types.SimpleNamespace(is_npu=lambda: npu), + parse_options=lambda values: dict(values), + resolve_model_reference=lambda request: (request.Model, False), + setup_parsers=lambda options: (None, None), + DiffusionParallelConfig=lambda **kwargs: kwargs, + backend_pb2=types.SimpleNamespace(Result=types.SimpleNamespace), + Omni=omni, + ) + exec(compile(ast.Module(body=[method], type_ignores=[]), '', 'exec'), namespace) + request = types.SimpleNamespace( + Model='test/model', Type=model_type, Options=list((options or {}).items()), + EnforceEager=False, GPUMemoryUtilization=0, TensorParallelSize=0, + TrustRemoteCode=False, MaxModelLen=0, + ) + instance = types.SimpleNamespace() + with contextlib.redirect_stderr(io.StringIO()): + result = namespace['LoadModel'](instance, request, None) + self.assertTrue(result.success, result.message) + self.assertEqual(len(calls), 1) + return calls[0] + + def test_default_tts_load_uses_pinned_constructor(self): + self.assertIsNone(self.load_model('tts')['stage_configs_path']) + + def test_custom_stage_config_reaches_omni(self): + self.assertEqual(self.load_model('llm', {'stage_configs_path': '/models/stages.yaml'}) + ['stage_configs_path'], '/models/stages.yaml') + + def test_diffusion_uses_platform_npu_capability(self): + for npu in [False, True]: + with self.subTest(npu=npu): + kwargs = self.load_model('image', npu=npu) + self.assertEqual(kwargs['vae_use_slicing'], npu) + self.assertEqual(kwargs['vae_use_tiling'], npu) + + +if __name__ == '__main__': + unittest.main() diff --git a/backend/python/vllm-omni/test_install.sh b/backend/python/vllm-omni/test_install.sh new file mode 100644 index 000000000..7f53258be --- /dev/null +++ b/backend/python/vllm-omni/test_install.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# Exercise the real installer without downloading packages or requiring a GPU. +set -eu +backend_dir=$(cd "$(dirname "$0")" && pwd) +test_dir=$(mktemp -d) +trap 'rm -rf "$test_dir"' EXIT +cp "$backend_dir/install.sh" "$test_dir/" +mkdir -p "$test_dir/common" "$test_dir/bin" +cat > "$test_dir/common/libbackend.sh" <<'EOF' +installRequirements() { echo base >> "$INSTALL_LOG"; } +runProtogen() { echo protogen >> "$INSTALL_LOG"; } +EOF +cat > "$test_dir/bin/mock" <<'EOF' +#!/bin/bash +printf '%s %s\n' "${0##*/}" "$*" >> "$INSTALL_LOG" +if [[ "$*" == *"install . "* ]]; then + # v0.14's package-data lists only generic stage configs. A regular wheel + # needs an explicit manifest rule to retain ROCm/NPU/XPU YAML resources. + if ! grep -qx 'recursive-include vllm_omni \*.yaml' MANIFEST.in; then + echo 'FAIL: platform stage configs are not included in the wheel' >&2 + exit 23 + fi + if [ "$TEST_ARCH" = aarch64 ]; then + ! grep -q fa3-fwd requirements/cuda.txt || exit 20 + ! grep -q fa3-fwd pyproject.toml || exit 21 + grep -q other-dependency requirements/cuda.txt || exit 22 + fi + printf '%s\n' "$PWD" > "$INSTALL_LOG.source" +fi +case ${0##*/} in +uname) echo "$TEST_ARCH" ;; +python) + # The harness checks installer ordering; real builds execute the metadata + # validation against the installed wheel using the backend interpreter. + [ -f "$INSTALL_LOG.source" ] || exit 24 + # Source egg-info must not shadow the installed distribution metadata. + [ "$*" = '-I -' ] || exit 25 + cat > "$INSTALL_LOG.validation" + ;; +git) + if [ "$1" = clone ]; then mkdir -p vllm-omni; fi + mkdir -p requirements + printf 'fa3-fwd==0.0.3\nother-dependency\n' > requirements/cuda.txt + printf '"fa3-fwd==0.0.1",\n' > pyproject.toml + ;; +esac +EOF +chmod +x "$test_dir/bin/mock" +for command in pip uv git uname python; do ln -s mock "$test_dir/bin/$command"; done +export PATH="$test_dir/bin:$PATH" INSTALL_LOG="$test_dir/install.log" +fail() { echo "FAIL: $*" >&2; cat "$INSTALL_LOG" >&2; exit 1; } +cd "$test_dir" +for profile in cublas12 hipblas cublas13 l4t13; do + export BUILD_PROFILE=$profile BUILD_TYPE=cublas TEST_ARCH=x86_64 + version=0.14.0 + revision=ed89c8b0436999e9210f11363f4eb512330a9dfa + case $profile in + hipblas) BUILD_TYPE=hipblas ;; + cublas13|l4t13) version=0.20.0; revision=4a24a517abc7769b1399ded594558a3fe8269872 ;; + esac + if [ "$profile" = l4t13 ]; then TEST_ARCH=aarch64; fi + for USE_PIP in false true; do + export USE_PIP + : > "$INSTALL_LOG" + rm -f "$INSTALL_LOG.source" "$INSTALL_LOG.validation" + bash install.sh + [ -s "$INSTALL_LOG.validation" ] || fail "$profile: installed resources were not checked" + grep -q "$revision" "$INSTALL_LOG" || fail "$profile: source is not pinned" + grep -q "vllm==$version" "$INSTALL_LOG" || fail "$profile: engine is not pinned" + if grep -q ' -e ' "$INSTALL_LOG"; then fail "$profile: editable package cannot relocate"; fi + if [ "$USE_PIP" = true ] && grep -q -- '--torch-backend' "$INSTALL_LOG"; then + fail "$profile: pip does not accept --torch-backend" + fi + last_install=$(grep ' install ' "$INSTALL_LOG" | tail -1) + case $last_install in *"vllm==$version"*) ;; *) fail "$profile: Omni resolution must retain engine pin" ;; esac + [ "$(tail -1 "$INSTALL_LOG")" = protogen ] || fail 'protobuf must regenerate after installation' + [ ! -e "$(cat "$INSTALL_LOG.source")" ] || fail "temporary source checkout was not removed" + grep -q "git tag v$version" "$INSTALL_LOG" || fail "release metadata is missing" + echo "PASS: $profile USE_PIP=$USE_PIP" + done +done diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md index 60786a22e..c95e18f6c 100644 --- a/docs/content/reference/compatibility-table.md +++ b/docs/content/reference/compatibility-table.md @@ -155,3 +155,23 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ Note: any backend name listed above can be used in the `backend` field of the model configuration file (See [the advanced section]({{%relref "advanced" %}})). - \* Only for CUDA and OpenVINO CPU/XPU acceleration. + +### vLLM Omni package versions + +The `vllm-omni` backend packages matched vLLM and vLLM Omni releases: + +| Build profile | vLLM | vLLM Omni source tag | +| --- | --- | --- | +| CUDA 12 and ROCm | 0.14.0 | v0.14.0 | +| CUDA 13 and Jetson L4T 13 (aarch64) | 0.20.0 | v0.20.0 | + +The installer pins each Omni source revision and installs a regular package. +The backend can therefore move from the build directory to its runtime directory +without requiring the original source checkout. These releases support the +`stage_configs_path` model option for LLM and TTS models. Do not replace Omni +with an unpinned upstream checkout: newer releases can remove this option or +change the vLLM APIs that the backend uses. + +The package includes platform-specific stage configurations, including ROCm +batching defaults. Installation checks that all upstream YAML resources are +present in the installed package before removing the source checkout.