mirror of
https://github.com/mudler/LocalAI.git
synced 2026-06-08 00:36:37 -04:00
2de6ca51d4
The L4T13 vllm backend pulled torch / torchvision / torchaudio / vllm from pypi.jetson-ai-lab.io's sbsa/cu130 mirror via [tool.uv.sources] with no version pins. That mirror started shipping torch 2.11.0 next to a vllm-0.20.0+cu130 wheel that was still compiled against torch 2.10's c10 ABI, so uv landed on the mismatched pair and vllm crashed at import: ImportError: vllm/_C.abi3.so: undefined symbol: _ZN3c1013MessageLoggerC1EPKciib (c10::MessageLogger's constructor signature changed between torch 2.10 and 2.11; the vllm wheel referenced the 2.10 form, the installed libc10.so exported only the 2.11 form.) Since torch 2.11 (April 2026) PyPI publishes its own aarch64 + cu130 manylinux wheels, and vllm 0.20.0 ships an aarch64 wheel whose Requires- Dist locks torch==2.11.0 / torchvision==0.26.0 / torchaudio==2.11.0. That makes uv's resolver produce an ABI-consistent set automatically, so the mirror and the [tool.uv.sources] pinning are no longer needed. flash-attn is dropped from the dep list: PyPI has no aarch64 wheel, but vLLM 0.20+ already bundles its own vllm_flash_attn (fa2 + fa3) inside the main wheel, so the Dao-AILab package isn't required at runtime. Reference: https://pytorch.org/blog/vllm-and-pytorch-work-together-to-improve-the-developer-experience-on-aarch64/ Assisted-by: Claude:claude-opus-4-7 [Read] [Edit] [Write] [Bash] [WebFetch] Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
54 lines
2.3 KiB
TOML
54 lines
2.3 KiB
TOML
# L4T arm64 (JetPack 7 / sbsa cu130) install spec for the vllm backend.
|
|
#
|
|
# Since PyTorch 2.11 (April 2026) PyPI publishes aarch64 + cu130 manylinux
|
|
# wheels directly for torch / torchvision / torchaudio, and vllm 0.20+ ships
|
|
# an aarch64 wheel whose Requires-Dist pins those exact versions. uv's
|
|
# resolver therefore locks an ABI-consistent set without any custom index.
|
|
# https://pytorch.org/blog/vllm-and-pytorch-work-together-to-improve-the-developer-experience-on-aarch64/
|
|
#
|
|
# Historically this file pinned torch / vllm / flash-attn / torchvision /
|
|
# torchaudio to pypi.jetson-ai-lab.io's SBSA cu130 mirror via
|
|
# [tool.uv.sources]. That mirror drifted out of sync (it published torch
|
|
# 2.11.0 next to a vllm wheel still built against torch 2.10's c10 ABI,
|
|
# producing `undefined symbol: _ZN3c1013MessageLoggerC1EPKciib` at import
|
|
# time). Moving to PyPI eliminates that drift class entirely.
|
|
#
|
|
# flash-attn is intentionally dropped: PyPI ships no aarch64 wheel for it,
|
|
# but vLLM 0.20+ already bundles its own vllm_flash_attn (fa2 + fa3)
|
|
# inside the main wheel, so the Dao-AILab package is not required at
|
|
# runtime.
|
|
#
|
|
# pyproject.toml (rather than requirements.txt) is still used on l4t13 so
|
|
# the build deps pass in requirements-install.txt - fastsafetensors's sdist
|
|
# needs pybind11 in the venv before --no-build-isolation can succeed - can
|
|
# run first; install.sh's l4t13 branch invokes `uv pip install --requirement
|
|
# pyproject.toml` after that pre-pass.
|
|
[project]
|
|
name = "localai-vllm-l4t13"
|
|
version = "0.0.0"
|
|
requires-python = ">=3.12,<3.13"
|
|
dependencies = [
|
|
# Mirror of requirements.txt - kept in sync manually for now since the
|
|
# l4t13 path bypasses installRequirements (see install.sh).
|
|
"grpcio==1.80.0",
|
|
"protobuf",
|
|
"certifi",
|
|
"setuptools",
|
|
"pillow",
|
|
"charset-normalizer>=3.4.7",
|
|
"chardet",
|
|
# Accelerator stack from PyPI (aarch64 + cu130 wheels). vllm's
|
|
# Requires-Dist locks torch==2.11.0 / torchvision==0.26.0 /
|
|
# torchaudio==2.11.0, so listing them unpinned here just lets the
|
|
# resolver echo those exact versions back.
|
|
"torch",
|
|
"torchvision",
|
|
"torchaudio",
|
|
"vllm",
|
|
# PyPI-resolvable packages that complete the runtime - accelerate,
|
|
# transformers, bitsandbytes carry their own wheels for aarch64.
|
|
"accelerate",
|
|
"transformers",
|
|
"bitsandbytes",
|
|
]
|